├── .gitignore ├── CPU_check ├── aws_lambda │ └── app.py └── gcp_run │ ├── Dockerfile │ ├── build.sh.sample │ ├── main.py │ └── requirements.txt ├── IaC ├── AWS-Lambda-IaC │ ├── README.md │ ├── data.tf │ ├── lambda │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── lambdas.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf.sample ├── AWS-Sagemaker-IaC │ ├── iam.tf │ ├── main.tf │ ├── outputs.tf │ ├── sagemaker │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── sagemakers.tf │ └── variables.tf.sample ├── Azure-Function-IaC │ ├── function │ │ ├── main.tf │ │ └── variables.tf │ ├── functions.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf.samples ├── GCP-Functions-IaC │ ├── cloudfunction │ │ ├── cloudfunction.tf │ │ ├── outputs.tf │ │ └── variables.tf │ ├── cloudfunctions.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf.sample └── GCP-Run-IaC │ ├── cloudrun │ ├── cloudrun.tf │ ├── outputs.tf │ └── variables.tf │ ├── cloudruns.tf │ ├── iam.tf │ ├── main.tf │ ├── outputs.tf │ └── variables.tf.sample ├── README.md ├── bench ├── bench_in_faas │ ├── aws_lambda │ │ ├── IaC │ │ │ ├── data.tf │ │ │ ├── lambda.tf │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf.sample │ │ ├── build_assets │ │ │ ├── .dockerignore │ │ │ ├── Dockerfile │ │ │ ├── app.py │ │ │ ├── build.sh.sample │ │ │ └── requirements.txt │ │ ├── go.mod │ │ ├── go.sum │ │ ├── python_backup │ │ │ ├── preprocess │ │ │ │ ├── bert_imdb.py │ │ │ │ ├── distilbert_sst2.py │ │ │ │ ├── inception_v3.py │ │ │ │ ├── mobilenet_v1.py │ │ │ │ ├── mobilenet_v2.py │ │ │ │ └── yolo_v5.py │ │ │ ├── run_bench.py │ │ │ └── variables.py.sample │ │ ├── run_bench.go │ │ ├── run_bench.sh │ │ └── variables.sh.samples │ ├── aws_sagemaker │ │ ├── IaC │ │ │ ├── data.tf │ │ │ ├── lambda.tf │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf.sample │ │ ├── build_assets │ │ │ ├── .dockerignore │ │ │ ├── Dockerfile │ │ │ ├── app.py │ │ │ ├── build.sh.sample │ │ │ └── requirements.txt │ │ ├── go.mod │ │ ├── go.sum │ │ ├── python_backup │ │ │ ├── preprocess │ │ │ │ ├── bert_imdb.py │ │ │ │ ├── distilbert_sst2.py │ │ │ │ ├── inception_v3.py │ │ │ │ ├── mobilenet_v1.py │ │ │ │ ├── mobilenet_v2.py │ │ │ │ └── yolo_v5.py │ │ │ ├── run_bench.py │ │ │ └── variables.py.sample │ │ ├── run_bench.go │ │ ├── run_bench.sh │ │ └── variables.sh.samples │ ├── gcp_run │ │ ├── gcp_functions_backup │ │ │ ├── grpc │ │ │ │ ├── IaC │ │ │ │ │ ├── cloudfunction.tf │ │ │ │ │ ├── main.tf │ │ │ │ │ ├── outputs.tf │ │ │ │ │ └── variables.tf.sample │ │ │ │ ├── README.md │ │ │ │ ├── build_assets │ │ │ │ │ ├── build.sh.sample │ │ │ │ │ ├── main.py │ │ │ │ │ └── requirements.txt │ │ │ │ ├── preprocess │ │ │ │ │ ├── bert_imdb.py │ │ │ │ │ ├── distilbert_sst2.py │ │ │ │ │ ├── inception_v3.py │ │ │ │ │ ├── mobilenet_v1.py │ │ │ │ │ ├── mobilenet_v2.py │ │ │ │ │ └── yolo_v5.py │ │ │ │ ├── run_bench.py │ │ │ │ └── variables.py.sample │ │ │ └── rest │ │ │ │ ├── IaC │ │ │ │ ├── cloudfunction.tf │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ └── variables.tf.sample │ │ │ │ ├── build_assets │ │ │ │ ├── build.sh.sample │ │ │ │ ├── main.py │ │ │ │ └── requirements.txt │ │ │ │ ├── preprocess │ │ │ │ ├── bert_imdb.py │ │ │ │ ├── distilbert_sst2.py │ │ │ │ ├── inception_v3.py │ │ │ │ ├── mobilenet_v1.py │ │ │ │ ├── mobilenet_v2.py │ │ │ │ └── yolo_v5.py │ │ │ │ ├── run_bench.py │ │ │ │ └── variables.py.sample │ │ ├── grpc │ │ │ ├── IaC │ │ │ │ ├── data.tf │ │ │ │ ├── lambda.tf │ │ │ │ ├── main.tf │ │ │ │ ├── outputs.tf │ │ │ │ └── variables.tf.sample │ │ │ ├── build_assets │ │ │ │ ├── .dockerignore │ │ │ │ ├── Dockerfile │ │ │ │ ├── app.py │ │ │ │ ├── build.sh.sample │ │ │ │ └── requirements.txt │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ ├── python_backup │ │ │ │ ├── preprocess │ │ │ │ │ ├── bert_imdb.py │ │ │ │ │ ├── distilbert_sst2.py │ │ │ │ │ ├── inception_v3.py │ │ │ │ │ ├── mobilenet_v1.py │ │ │ │ │ ├── mobilenet_v2.py │ │ │ │ │ └── yolo_v5.py │ │ │ │ ├── run_bench.py │ │ │ │ └── variables.py.sample │ │ │ ├── run_bench.go │ │ │ ├── run_bench.sh │ │ │ └── variables.sh.samples │ │ └── rest │ │ │ ├── IaC │ │ │ ├── data.tf │ │ │ ├── lambda.tf │ │ │ ├── main.tf │ │ │ ├── outputs.tf │ │ │ └── variables.tf.sample │ │ │ ├── build_assets │ │ │ ├── .dockerignore │ │ │ ├── Dockerfile │ │ │ ├── app.py │ │ │ ├── build.sh.sample │ │ │ └── requirements.txt │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ ├── python_backup │ │ │ ├── preprocess │ │ │ │ ├── bert_imdb.py │ │ │ │ ├── distilbert_sst2.py │ │ │ │ ├── inception_v3.py │ │ │ │ ├── mobilenet_v1.py │ │ │ │ ├── mobilenet_v2.py │ │ │ │ └── yolo_v5.py │ │ │ ├── run_bench.py │ │ │ └── variables.py.sample │ │ │ ├── run_bench.go │ │ │ ├── run_bench.sh │ │ │ └── variables.sh.samples │ └── generate_request_data │ │ └── rest │ │ ├── .gitignore │ │ ├── bert_imdb.py │ │ ├── distilbert_sst2.py │ │ ├── inception_v3.py │ │ ├── mobilenet_v1.py │ │ ├── mobilenet_v2.py │ │ └── yolo_v5.py ├── bench_in_instance │ ├── README.md │ ├── aws_lambda_variables.py.sample │ ├── azure_function_variables.py.sample │ ├── bench_aws_infra │ │ ├── data.tf │ │ ├── ec2.tf │ │ ├── iam.tf │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── variables.tf.sample │ │ └── vpc.tf │ ├── bench_azure_infra │ │ ├── main.tf │ │ ├── network.tf │ │ ├── outputs.tf │ │ ├── script.sh │ │ ├── variables.tf.sample │ │ └── vm.tf │ ├── bench_gcp_infra │ │ ├── README.md │ │ ├── instance.tf │ │ ├── main.tf │ │ ├── variables.tf.sample │ │ └── vpc.tf │ ├── bert_imdb │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py │ ├── distilbert_sst2 │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py │ ├── gcp_functions_variables.py.sample │ ├── gcp_run_variables.py.sample │ ├── inception_v3 │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py │ ├── logs_variables.py.sample │ ├── mobilenet_v1 │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py │ ├── mobilenet_v2 │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py │ ├── module │ │ ├── module_cw_logs.py │ │ ├── module_faas.py │ │ ├── module_grpc.py │ │ ├── module_grpc_each_session.py │ │ ├── module_gspread.py │ │ ├── module_rest.py │ │ ├── module_rest.py.gzip │ │ ├── put_data_into_sheet.py │ │ ├── run_faas_bench.py │ │ ├── run_grpc_bench.py │ │ ├── run_grpc_each_session_bench.py │ │ └── run_rest_bench.py │ ├── run_aws_lambda_bench.py │ ├── run_azure_function_bench.py │ ├── run_gcp_functions_bench.py │ ├── run_gcp_run_grpc_bench.py │ ├── run_gcp_run_grpc_each_session_bench.py │ ├── run_gcp_run_rest_bench.py │ └── yolo_v5 │ │ ├── faas_bench.py │ │ ├── grpc_bench.py │ │ ├── grpc_each_session_bench.py │ │ ├── preprocessing.py │ │ └── rest_bench.py └── requirements.txt ├── build_assets ├── aws_lambda_dockerfiles │ ├── .dockerignore │ ├── Dockerfile │ ├── bert_imdb │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── app.py │ ├── distilbert_sst2 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── app.py │ ├── inception_v3 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── app.py │ ├── mobilenet_v1 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── app.py │ ├── mobilenet_v2 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── app.py │ ├── requirements.txt │ └── yolo_v5 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ ├── app.py │ │ ├── app.py.bak │ │ └── app.py.gzip ├── aws_sagemaker_dockerfiles │ ├── Dockerfile │ └── requirements.txt ├── azure_function_dockerfiles │ ├── .dockerignore │ ├── Dockerfile │ ├── bert_imdb │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── __init__.py │ ├── function.json │ ├── host.json │ ├── inception_v3 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── __init__.py │ ├── mobilenet_v1 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── __init__.py │ ├── mobilenet_v2 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── __init__.py │ ├── requirements.txt │ └── yolo_v5 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── __init__.py ├── gcp_functions_codes │ ├── bert_imdb │ │ └── main.py │ ├── distilbert_sst2 │ │ └── main.py │ ├── inception_v3 │ │ └── main.py │ ├── mobilenet_v1 │ │ └── main.py │ ├── mobilenet_v2 │ │ └── main.py │ ├── requirements.txt │ └── yolo_v5 │ │ └── main.py ├── gcp_run_dockerfiles │ ├── bert_imdb │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest │ ├── distilbert_sst2 │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest │ ├── inception_v3 │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest │ ├── mobilenet_v1 │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest │ ├── mobilenet_v2 │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest │ └── yolo_v5 │ │ ├── .dockerignore │ │ ├── Dockerfile.grpc │ │ └── Dockerfile.rest └── gcp_run_python_dockerfiles │ ├── .dockerignore.grpc │ ├── .dockerignore.grpc.monitoring │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── bert_imdb │ ├── .dockerignore.grpc │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── grpc_main.py │ └── rest_main.py │ ├── grpc_start.sh │ ├── inception_v3 │ ├── .dockerignore.grpc │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── grpc_main.py │ └── rest_main.py │ ├── mobilenet_v1 │ ├── .dockerignore.grpc │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── grpc_main.py │ └── rest_main.py │ ├── mobilenet_v2 │ ├── .dockerignore.grpc │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── grpc_main.py │ └── rest_main.py │ ├── push_metrics.py │ ├── requirements.grpc.monitoring.txt │ ├── requirements.grpc.txt │ ├── requirements.rest.txt │ └── yolo_v5 │ ├── .dockerignore.grpc │ ├── .dockerignore.rest │ ├── Dockerfile.grpc │ ├── Dockerfile.grpc.monitoring │ ├── Dockerfile.rest │ ├── grpc_main.py │ └── rest_main.py ├── build_aws_lambda_docker_image.sh.sample ├── build_aws_sagemaker_docker_image.sh.sample ├── build_azure_function_docker_image.sh.sample ├── build_gcp_run_docker_image.sh.sample ├── build_gcp_run_python_docker_image.sh.sample ├── create_gcp_functions_code_zip.sh.sample ├── dataset ├── README.md ├── dataset_download.sh └── requirements.txt └── models ├── README.md ├── model_download.sh └── sagemaker_assets ├── grpc ├── inception_v3 │ └── inference.py ├── inference.py └── yolo_v5 │ └── inference.py └── rest ├── inception_v3 └── inference.py ├── inference.py └── yolo_v5 └── inference.py /CPU_check/aws_lambda/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import multiprocessing 3 | import subprocess 4 | import json 5 | 6 | def lambda_handler(event,context): 7 | mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') 8 | mem_gib = mem_bytes/(1024.**3) 9 | num_cores = multiprocessing.cpu_count() 10 | cpu_info = subprocess.check_output ('cat /proc/cpuinfo', shell=True) 11 | mem_info = subprocess.check_output('cat /proc/meminfo', shell=True) 12 | 13 | response = { 14 | "statusCode": 200, 15 | "headers": { 16 | "Content-Type": "application/json" 17 | }, 18 | "body": { 19 | "mem_bytes": mem_bytes, 20 | "mem_gib": mem_gib, 21 | "num_cores": num_cores, 22 | "cpu_info": cpu_info, 23 | "mem_info": mem_info 24 | } 25 | } 26 | return response 27 | -------------------------------------------------------------------------------- /CPU_check/gcp_run/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.11 2 | 3 | WORKDIR /app 4 | 5 | COPY ./requirements.txt /app 6 | 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | COPY ./main.py /app 10 | 11 | EXPOSE 8501 12 | 13 | CMD ["uvicorn", "main:app", "--port", "8501", "--host", "0.0.0.0"] 14 | 15 | -------------------------------------------------------------------------------- /CPU_check/gcp_run/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #you need to login Artifact Registry (docker login) 3 | DOCKER_REGISTRY="" 4 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/scpc_run_cpu_check:latest -f ./Dockerfile . --build-arg DOCKERIGNORE_FILE=./.dockerignore 5 | sudo docker builder prune -f 6 | sudo docker push $DOCKER_REGISTRY/scpc_run_cpu_check:latest -------------------------------------------------------------------------------- /CPU_check/gcp_run/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import multiprocessing 3 | import subprocess 4 | from fastapi import FastAPI 5 | app = FastAPI() 6 | 7 | @app.get('/') 8 | async def main(): 9 | mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') 10 | mem_gib = mem_bytes/(1024.**3) 11 | num_cores = multiprocessing.cpu_count() 12 | cpu_info = subprocess.check_output ('cat /proc/cpuinfo', shell=True) 13 | mem_info = subprocess.check_output('cat /proc/meminfo', shell=True) 14 | 15 | response = { 16 | "mem_bytes": mem_bytes, 17 | "mem_gib": mem_gib, 18 | "num_cores": num_cores, 19 | "cpu_info": cpu_info, 20 | "mem_info": mem_info 21 | } 22 | return response 23 | -------------------------------------------------------------------------------- /CPU_check/gcp_run/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi==0.95.2 2 | uvicorn[standard]==0.22.0 -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/README.md: -------------------------------------------------------------------------------- 1 | ### Route53 2 | - You need create ACM validate CNAME record -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_route53_zone" "route53_zone" { 2 | name = "${var.route53_domain}." 3 | } -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/lambda/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | value = aws_apigatewayv2_domain_name.api_domain_name.domain_name 3 | } -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/lambda/variables.tf: -------------------------------------------------------------------------------- 1 | variable "prefix" {} 2 | variable "model_name" {} 3 | variable "docker_registry" {} 4 | variable "route53_zoneid" {} 5 | variable "route53_domain" {} 6 | variable "ram_mib" {} 7 | -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/lambdas.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_bucket" "bucket" { 2 | bucket = "${var.prefix}-scpc-bucket" 3 | force_destroy = true 4 | } 5 | 6 | module "lambda" { 7 | count = length(var.enabled_models) 8 | source = "./lambda" 9 | prefix = var.prefix 10 | docker_registry = var.docker_registry 11 | route53_zoneid = data.aws_route53_zone.route53_zone.zone_id 12 | route53_domain = var.route53_domain 13 | model_name = var.enabled_models[count.index] 14 | ram_mib = var.model_resources[var.enabled_models[count.index]].ram_mib 15 | } 16 | -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url_output" { 2 | value = module.lambda[*].api_gateway_url 3 | } 4 | 5 | output "bucket_name" { 6 | value = "${var.prefix}-scpc-bucket" 7 | } -------------------------------------------------------------------------------- /IaC/AWS-Lambda-IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "route53_domain" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "docker_registry" { 22 | type = string 23 | default = "change here!" 24 | } 25 | 26 | variable "enabled_models" { 27 | type = list(string) 28 | default = [ "mobilenet_v1", "mobilenet_v2", "inception_v3", "yolo_v5", "bert_imdb"] 29 | } 30 | 31 | variable "model_resources" { 32 | type = object({ 33 | mobilenet_v1 = object({ 34 | ram_mib = number 35 | }) 36 | mobilenet_v2 = object({ 37 | ram_mib = number 38 | }) 39 | inception_v3 = object({ 40 | ram_mib = number 41 | }) 42 | yolo_v5 = object({ 43 | ram_mib = number 44 | }) 45 | bert_imdb = object({ 46 | ram_mib = number 47 | }) 48 | distilbert_sst2 = object({ 49 | ram_mib = number 50 | }) 51 | }) 52 | default = { 53 | mobilenet_v1 = { 54 | ram_mib = 4096 55 | } 56 | mobilenet_v2 = { 57 | ram_mib = 4096 58 | } 59 | inception_v3 = { 60 | ram_mib = 4096 61 | } 62 | yolo_v5 = { 63 | ram_mib = 4096 64 | } 65 | bert_imdb = { 66 | ram_mib = 4096 67 | } 68 | distilbert_sst2 = { 69 | ram_mib = 4096 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/iam.tf: -------------------------------------------------------------------------------- 1 | 2 | data "aws_iam_policy_document" "sagemaker-assume-role" { 3 | statement { 4 | principals { 5 | type = "Service" 6 | identifiers = ["sagemaker.amazonaws.com"] 7 | } 8 | actions = ["sts:AssumeRole"] 9 | } 10 | } 11 | 12 | resource "aws_iam_role" "sagemaker-role" { 13 | name = "${var.prefix}-scpc-sagemaker-role" 14 | assume_role_policy = data.aws_iam_policy_document.sagemaker-assume-role.json 15 | } 16 | 17 | resource "aws_iam_role_policy_attachment" "sagemaker-full-access-policy" { 18 | role = aws_iam_role.sagemaker-role.name 19 | policy_arn = "arn:aws:iam::aws:policy/AmazonSageMakerFullAccess" 20 | } 21 | 22 | resource "aws_iam_role_policy_attachment" "cloudwatch-full-access-policy" { 23 | role = aws_iam_role.sagemaker-role.name 24 | policy_arn = "arn:aws:iam::aws:policy/CloudWatchFullAccess" 25 | } 26 | 27 | resource "aws_iam_role_policy_attachment" "cloudwatch-event-full-access-policy" { 28 | role = aws_iam_role.sagemaker-role.name 29 | policy_arn = "arn:aws:iam::aws:policy/CloudWatchEventsFullAccess" 30 | } 31 | 32 | resource "aws_iam_role_policy_attachment" "cloudwatch-logs-full-access-policy" { 33 | role = aws_iam_role.sagemaker-role.name 34 | policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" 35 | } 36 | 37 | resource "aws_iam_role_policy_attachment" "s3-full-access-policy" { 38 | role = aws_iam_role.sagemaker-role.name 39 | policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess" 40 | } 41 | -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "endpoint_arn_output" { 2 | value = module.sagemaker[*].endpoint_arn 3 | } -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/sagemaker/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_sagemaker_model" "model" { 2 | name = "${var.prefix}-scpc-${replace(var.model_name, "_", "-")}-model" 3 | execution_role_arn = var.iam_role 4 | 5 | primary_container { 6 | image = var.sagemaker_prebuilt_image 7 | model_data_url = "s3://${var.model_bucket}/${var.model_name}.tgz" 8 | } 9 | } 10 | 11 | resource "aws_sagemaker_endpoint_configuration" "endpoint_configuration" { 12 | name = "${var.prefix}-scpc-${replace(var.model_name, "_", "-")}-endpoint-configuration" 13 | 14 | production_variants { 15 | variant_name = "default" 16 | model_name = aws_sagemaker_model.model.name 17 | serverless_config { 18 | max_concurrency = var.max_concurrency 19 | provisioned_concurrency = var.provisioned_concurrency > 0 ? var.provisioned_concurrency : null 20 | memory_size_in_mb = var.ram_mib 21 | } 22 | } 23 | } 24 | 25 | resource "aws_sagemaker_endpoint" "endpoint" { 26 | name = "${var.prefix}-${replace(var.model_name, "_", "-")}-endpoint" 27 | endpoint_config_name = aws_sagemaker_endpoint_configuration.endpoint_configuration.name 28 | lifecycle { 29 | replace_triggered_by = [ aws_sagemaker_endpoint_configuration.endpoint_configuration ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/sagemaker/outputs.tf: -------------------------------------------------------------------------------- 1 | output "endpoint_arn" { 2 | value = aws_sagemaker_endpoint.endpoint.arn 3 | } -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/sagemaker/variables.tf: -------------------------------------------------------------------------------- 1 | variable "prefix" {} 2 | variable "model_name" {} 3 | variable "sagemaker_prebuilt_image" {} 4 | variable "ram_mib" {} 5 | variable "max_concurrency" {} 6 | variable "provisioned_concurrency" {} 7 | variable "model_bucket" {} 8 | variable "iam_role" {} 9 | -------------------------------------------------------------------------------- /IaC/AWS-Sagemaker-IaC/sagemakers.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_bucket" "bucket" { 2 | bucket = "${var.prefix}-scpc-sagemaker-bucket" 3 | force_destroy = true 4 | } 5 | 6 | module "sagemaker" { 7 | count = length(var.enabled_models) 8 | source = "./sagemaker" 9 | prefix = var.prefix 10 | max_concurrency = var.max_concurrency 11 | provisioned_concurrency = var.provisioned_concurrency 12 | model_bucket = var.model_bucket 13 | model_name = var.enabled_models[count.index] 14 | sagemaker_prebuilt_image = var.sagemaker_prebuilt_image 15 | ram_mib = var.model_resources[var.enabled_models[count.index]].ram_mib 16 | iam_role = aws_iam_role.sagemaker-role.arn 17 | } 18 | -------------------------------------------------------------------------------- /IaC/Azure-Function-IaC/function/variables.tf: -------------------------------------------------------------------------------- 1 | variable "resource_group" {} 2 | variable "model_name" {} 3 | variable "prefix" {} 4 | variable "docker_registry" {} 5 | variable app_service_tier {} 6 | variable app_service_size {} 7 | variable "index_num" {} 8 | variable "BlobStorageConnectionString" {} -------------------------------------------------------------------------------- /IaC/Azure-Function-IaC/functions.tf: -------------------------------------------------------------------------------- 1 | module "function" { 2 | count = length(var.enabled_models) 3 | index_num = count.index 4 | source = "./function" 5 | resource_group = data.azurerm_resource_group.bench-resource-group 6 | prefix = var.prefix 7 | docker_registry = data.azurerm_container_registry.bench-container-registry 8 | model_name = var.enabled_models[count.index] 9 | app_service_tier = var.model_resources[var.enabled_models[count.index]].tier 10 | app_service_size = var.model_resources[var.enabled_models[count.index]].size 11 | BlobStorageConnectionString = azurerm_storage_account.storage-account.primary_connection_string 12 | } 13 | -------------------------------------------------------------------------------- /IaC/Azure-Function-IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "azurerm" { 2 | features {} 3 | } 4 | 5 | data "azurerm_resource_group" "bench-resource-group" { 6 | name = var.resource_group_name 7 | } 8 | 9 | data "azurerm_container_registry" "bench-container-registry" { 10 | name = var.acr_registry_name 11 | resource_group_name = var.resource_group_name 12 | } 13 | 14 | resource "azurerm_storage_account" "storage-account" { 15 | name = "${var.prefix}scpcblobsa" 16 | location = data.azurerm_resource_group.bench-resource-group.location 17 | resource_group_name = data.azurerm_resource_group.bench-resource-group.name 18 | account_tier = "Standard" 19 | account_replication_type = "LRS" 20 | } 21 | 22 | resource "azurerm_storage_container" "storage-container" { 23 | name = "${var.prefix}scpcstoragecontainer" 24 | storage_account_name = azurerm_storage_account.storage-account.name 25 | container_access_type = "private" 26 | } 27 | -------------------------------------------------------------------------------- /IaC/Azure-Function-IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "blob_connection_string" { 2 | value = nonsensitive(azurerm_storage_account.storage-account.primary_connection_string) 3 | } 4 | 5 | output "blob_container_name" { 6 | value = azurerm_storage_container.storage-container.name 7 | } -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/cloudfunction/cloudfunction.tf: -------------------------------------------------------------------------------- 1 | resource "google_cloudfunctions_function" "cloudfunctions_function" { 2 | name = "function-${replace(var.model_name, "_", "-")}" 3 | region = var.region 4 | runtime = "python310" 5 | source_archive_bucket = var.bucket_name 6 | source_archive_object = "${var.model_name}.zip" 7 | entry_point = "predict" 8 | available_memory_mb = var.ram_mib 9 | min_instances = var.min_instances 10 | max_instances = var.max_instances 11 | timeout = 120 12 | trigger_http = true 13 | https_trigger_security_level = "SECURE_ALWAYS" 14 | 15 | timeouts { 16 | create = "15m" 17 | } 18 | } 19 | 20 | resource "google_cloudfunctions_function_iam_member" "cloudfunction_noauth" { 21 | region = google_cloudfunctions_function.cloudfunctions_function.region 22 | project = google_cloudfunctions_function.cloudfunctions_function.project 23 | cloud_function = google_cloudfunctions_function.cloudfunctions_function.name 24 | role = "roles/cloudfunctions.invoker" 25 | member = "allUsers" 26 | } 27 | -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/cloudfunction/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloudfunction_url" { 2 | value = google_cloudfunctions_function.cloudfunctions_function.https_trigger_url 3 | } 4 | -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/cloudfunction/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" {} 2 | variable "model_name" {} 3 | variable "bucket_name" {} 4 | variable "ram_mib" {} 5 | variable "min_instances" {} 6 | variable "max_instances" {} 7 | -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/cloudfunctions.tf: -------------------------------------------------------------------------------- 1 | # policy_data = data.google_iam_policy.noauth.policy_data 2 | 3 | module "cloudfunction" { 4 | count = length(var.enabled_models) 5 | source = "./cloudfunction" 6 | region = var.region 7 | model_name = var.enabled_models[count.index] 8 | bucket_name = var.bucket_name 9 | ram_mib = var.model_resources[var.enabled_models[count.index]].ram_mib 10 | min_instances = var.model_resources[var.enabled_models[count.index]].min_instances 11 | max_instances = var.model_resources[var.enabled_models[count.index]].max_instances 12 | } 13 | -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.project 3 | region = var.region 4 | } 5 | -------------------------------------------------------------------------------- /IaC/GCP-Functions-IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_function_url_output" { 2 | value = module.cloudfunction[*].cloudfunction_url 3 | } -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/cloudrun/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloudrun_url" { 2 | value = google_cloud_run_service.cloudrun_service[*].status[0].url 3 | } -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/cloudrun/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" {} 2 | variable "model_name" {} 3 | variable "docker_registry" {} 4 | variable "cpu_number" {} 5 | variable "ram_mib" {} 6 | variable "min_instances" {} 7 | variable "max_instances" {} 8 | variable "noauth_policy" {} 9 | variable "APIS" {} 10 | variable "prefix" {} 11 | variable "concurrency" {} 12 | variable "project_name" {} 13 | variable "pushgateway_address" {} -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/cloudruns.tf: -------------------------------------------------------------------------------- 1 | # policy_data = data.google_iam_policy.noauth.policy_data 2 | 3 | module "cloudrun" { 4 | count = length(var.enabled_models) 5 | source = "./cloudrun" 6 | region = var.region 7 | model_name = var.enabled_models[count.index] 8 | docker_registry = var.docker_registry 9 | cpu_number = var.model_resources[var.enabled_models[count.index]].cpu_number 10 | ram_mib = var.model_resources[var.enabled_models[count.index]].ram_mib 11 | min_instances = var.model_resources[var.enabled_models[count.index]].min_instances 12 | max_instances = var.model_resources[var.enabled_models[count.index]].max_instances 13 | noauth_policy = data.google_iam_policy.noauth.policy_data 14 | APIS = var.APIS 15 | concurrency = var.concurrency 16 | project_name = var.project 17 | prefix = var.prefix 18 | pushgateway_address = var.pushgateway_address 19 | } 20 | -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/iam.tf: -------------------------------------------------------------------------------- 1 | 2 | data "google_iam_policy" "noauth" { 3 | binding { 4 | role = "roles/run.invoker" 5 | members = [ 6 | "allUsers", 7 | ] 8 | } 9 | } -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.project 3 | region = var.region 4 | } 5 | -------------------------------------------------------------------------------- /IaC/GCP-Run-IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_run_url_output" { 2 | value = module.cloudrun[*].cloudrun_url 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/IaC/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_route53_zone" "route53_zone" { 2 | name = "${var.route53_domain}." 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | value = aws_apigatewayv2_domain_name.api_domain_name.domain_name 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "route53_domain" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "docker_registry" { 22 | type = string 23 | default = "change here!" 24 | } 25 | 26 | variable "docker_image_tag" { 27 | type = string 28 | default = "change here!" 29 | } 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/build_assets/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !./app.py 3 | !./requirements.txt 4 | !./Dockerfile 5 | !./*.json -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/build_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.10 2 | COPY requirements.txt / 3 | RUN pip3 install --no-cache-dir -r /requirements.txt --target "${LAMBDA_TASK_ROOT}" 4 | COPY app.py ${LAMBDA_TASK_ROOT} 5 | COPY *.json ${LAMBDA_TASK_ROOT} 6 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com/bench-lambda 3 | #you need to create ECR repo and ecr login in instnace 4 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" "distilbert_sst2" ) 5 | for model_name in "${model_names[@]}" 6 | do 7 | if [ ! -e "${model_name}.json" ]; then 8 | curl -o "${model_name}.json" https://ddps-lab-scpc-bucket.s3.ap-northeast-2.amazonaws.com/rest/$model_name.json 9 | fi 10 | done 11 | 12 | DOCKER_REPOSITORY_NAME="" 13 | 14 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REPOSITORY_NAME:latest -f ./Dockerfile . --build-arg DOCKERIGNORE_FILE=./dockerignore 15 | sudo docker builder prune -f 16 | sudo docker push $DOCKER_REPOSITORY_NAME:latest -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/go.mod: -------------------------------------------------------------------------------- 1 | module run_bench.go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2/config v1.18.27 7 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 8 | github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 9 | github.com/awsdocs/aws-doc-sdk-examples/gov2/s3 v0.0.0-20230901162301-dec41fb58904 10 | ) 11 | 12 | require ( 13 | github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect 14 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect 15 | github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect 16 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect 17 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.38 // indirect 18 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect 20 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect 21 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect 22 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect 23 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect 24 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect 27 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect 29 | github.com/aws/smithy-go v1.14.2 // indirect 30 | github.com/jmespath/go-jmespath v0.4.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | input_ids = tokenizer.texts_to_sequences([text]) 12 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 13 | input_masks = [[1] * len(input_ids[0])] 14 | segment_ids = [[0] * len(input_ids[0])] 15 | return input_ids, input_masks, segment_ids 16 | 17 | def create_request_data(bucket_name=""): 18 | text = "This is a sample sentence to test the BERT model." 19 | input_ids, input_masks, segment_ids = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 21 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | bert_input_ids = tokenizer.texts_to_sequences([text]) 12 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 13 | bert_input_masks = [[1] * len(bert_input_ids[0])] 14 | 15 | return bert_input_ids, bert_input_masks 16 | 17 | def create_request_data(bucket_name=""): 18 | text = "This is a sample sentence to test the BERT model." 19 | bert_input_ids, bert_input_masks = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 21 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for inception) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((299, 299)) 14 | img_array = np.array(img) 15 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 16 | img_array = img_array.astype(np.float32) 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array 19 | 20 | def create_request_data(bucket_name=""): 21 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 22 | data = json.dumps({"inputs": { "input_3": run_preprocessing(image_file_path).tolist()}}) 23 | return data, 0 24 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(bucket_name=""): 20 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_1": run_preprocessing(image_file_path).tolist()}}) 22 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(bucket_name=""): 20 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_2": run_preprocessing(image_file_path).tolist()}}) 22 | return data, 0 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | import cv2 5 | import json 6 | import boto3 7 | import time 8 | 9 | def get_file_path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | # 이미지 로드 및 전처리 (for yolo) 13 | def run_preprocessing(image_file_path): 14 | img = cv2.imread(get_file_path(image_file_path)) 15 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 16 | img = cv2.resize(img, (640, 640)) 17 | img = img.astype('float32') / 255.0 18 | img = np.expand_dims(img, axis=0) 19 | return img 20 | 21 | def create_request_data(bucket_name=""): 22 | image_file_path = "../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 23 | 24 | np.save('preprocessed_data', run_preprocessing(image_file_path)) 25 | s3 = boto3.resource('s3') 26 | upload_start_time = time.time() 27 | s3.Bucket(bucket_name).upload_file('./preprocessed_data.npy', "preprocessed_data.npy") 28 | upload_time = time.time() - upload_start_time 29 | data = json.dumps({"inputs": {"s3_bucket_name": bucket_name, "s3_object_name": "preprocessed_data.npy"}}) 30 | return data, upload_time 31 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/python_backup/variables.py.sample: -------------------------------------------------------------------------------- 1 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) inference.com 3 | aws_lambda_address = '' 4 | # aws_lambda bench default address 5 | aws_lambda_default_address = '' 6 | # yolo_v5 추론 시 사용할 S3 Bucket name 7 | bucket_name = '' 8 | # 추론을 테스트할 모델 이름들 9 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 10 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 11 | num_tasks = [1,10,30] 12 | # CloudWatch Logs Log Group Name 13 | log_group_name = '' -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/run_bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./variables.sh 3 | 4 | # warm bench request lambda instances 5 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --aws_lambda_default_address "unknown" --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path 6 | sleep 5 7 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --aws_lambda_default_address "unknown" --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path 8 | sleep 5 9 | for model_name in "${model_names[@]}" 10 | do 11 | for task_num in "${task_nums[@]}" 12 | do 13 | go run run_bench.go --model_name $model_name --task_num $task_num --log_group_name $log_group_name --server_address $server_address --aws_lambda_default_address $aws_lambda_default_address --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path 14 | sleep 30 15 | done 16 | done -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_lambda/variables.sh.samples: -------------------------------------------------------------------------------- 1 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) inference.com 3 | server_address="" 4 | # aws_lambda bench default address 5 | aws_lambda_default_address="" 6 | # 추론을 테스트할 모델 이름들 7 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 8 | # 테스트 추론 횟수들 지정 (각 요쳥 횟수별 병렬로 한번에 처리됩니다.) 9 | task_nums=( 10 20 30 ) 10 | # S3 Bucket Name 11 | s3_bucket_name="" 12 | # Bucket url with preprocessed_data (s3://$s3_bucket_name$s3_folder_name/$model_name.json) 13 | s3_preprocessed_data_key_path="/" 14 | # CloudWatch Logs Log Group Name 15 | log_group_name="" -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/IaC/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_route53_zone" "route53_zone" { 2 | name = "${var.route53_domain}." 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | value = aws_apigatewayv2_domain_name.api_domain_name.domain_name 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "route53_domain" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "docker_registry" { 22 | type = string 23 | default = "change here!" 24 | } 25 | 26 | variable "docker_image_tag" { 27 | type = string 28 | default = "change here!" 29 | } 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/build_assets/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !./app.py 3 | !./requirements.txt 4 | !./Dockerfile 5 | !./*.json 6 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/build_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.10 2 | COPY requirements.txt / 3 | RUN pip3 install --no-cache-dir -r /requirements.txt --target "${LAMBDA_TASK_ROOT}" 4 | COPY app.py ${LAMBDA_TASK_ROOT} 5 | COPY *.json ${LAMBDA_TASK_ROOT} 6 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com/bench-lambda 3 | #you need to create ECR repo and ecr login in instnace 4 | model_names=( "mobilenet_v1" "bert_imdb" ) 5 | for model_name in "${model_names[@]}" 6 | do 7 | if [ ! -e "${model_name}-rest.json" ]; then 8 | curl -o "${model_name}-rest.json" https://ddps-lab-scpc-bucket.s3.ap-northeast-2.amazonaws.com/rest/$model_name.json 9 | fi 10 | if [ ! -e "${model_name}-grpc.json" ]; then 11 | curl -o "${model_name}-grpc.json" https://ddps-lab-scpc-bucket.s3.ap-northeast-2.amazonaws.com/sagemaker-grpc/$model_name.json 12 | fi 13 | 14 | done 15 | DOCKER_REPOSITORY_NAME="" 16 | 17 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REPOSITORY_NAME:latest -f ./Dockerfile . --build-arg DOCKERIGNORE_FILE=./dockerignore 18 | sudo docker builder prune -f 19 | sudo docker push $DOCKER_REPOSITORY_NAME:latest -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 3 | sagemaker -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/go.mod: -------------------------------------------------------------------------------- 1 | module run_bench.go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2/config v1.18.27 7 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 8 | github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 9 | github.com/awsdocs/aws-doc-sdk-examples/gov2/s3 v0.0.0-20230901133849-e95095a2511a 10 | ) 11 | 12 | require ( 13 | github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect 14 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect 15 | github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect 16 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect 17 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.38 // indirect 18 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect 20 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect 21 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect 22 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect 23 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect 24 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect 27 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect 29 | github.com/aws/smithy-go v1.14.2 // indirect 30 | github.com/jmespath/go-jmespath v0.4.0 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | input_ids = tokenizer.texts_to_sequences([text]) 12 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 13 | input_masks = [[1] * len(input_ids[0])] 14 | segment_ids = [[0] * len(input_ids[0])] 15 | return input_ids, input_masks, segment_ids 16 | 17 | def create_request_data(bucket_name=""): 18 | text = "This is a sample sentence to test the BERT model." 19 | input_ids, input_masks, segment_ids = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 21 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | bert_input_ids = tokenizer.texts_to_sequences([text]) 12 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 13 | bert_input_masks = [[1] * len(bert_input_ids[0])] 14 | 15 | return bert_input_ids, bert_input_masks 16 | 17 | def create_request_data(bucket_name=""): 18 | text = "This is a sample sentence to test the BERT model." 19 | bert_input_ids, bert_input_masks = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 21 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | import boto3 7 | import time 8 | 9 | def get_file_path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | # 이미지 로드 및 전처리 (for inception) 13 | def run_preprocessing(image_file_path): 14 | img = Image.open(get_file_path(image_file_path)) 15 | img = img.resize((299, 299)) 16 | img_array = np.array(img) 17 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 18 | img_array = img_array.astype(np.float32) 19 | img_array = np.expand_dims(img_array, axis=0) 20 | return img_array 21 | 22 | def create_request_data(bucket_name=""): 23 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 24 | 25 | np.save('preprocessed_data', run_preprocessing(image_file_path)) 26 | s3 = boto3.resource('s3') 27 | upload_start_time = time.time() 28 | s3.Bucket(bucket_name).upload_file('./preprocessed_data.npy', "preprocessed_data.npy") 29 | upload_time = time.time() - upload_start_time 30 | data = json.dumps({"inputs": {"s3_bucket_name": bucket_name, "s3_object_name": "preprocessed_data.npy"}}) 31 | return data, upload_time 32 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(bucket_name=""): 20 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_1": run_preprocessing(image_file_path).tolist()}}) 22 | return data, 0 -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(bucket_name=""): 20 | image_file_path = "../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_2": run_preprocessing(image_file_path).tolist()}}) 22 | return data, 0 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | import cv2 5 | import json 6 | import boto3 7 | import time 8 | 9 | def get_file_path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | # 이미지 로드 및 전처리 (for yolo) 13 | def run_preprocessing(image_file_path): 14 | img = cv2.imread(get_file_path(image_file_path)) 15 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 16 | img = cv2.resize(img, (640, 640)) 17 | img = img.astype('float32') / 255.0 18 | img = np.expand_dims(img, axis=0) 19 | return img 20 | 21 | def create_request_data(bucket_name=""): 22 | image_file_path = "../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 23 | 24 | np.save('preprocessed_data', run_preprocessing(image_file_path)) 25 | s3 = boto3.resource('s3') 26 | upload_start_time = time.time() 27 | s3.Bucket(bucket_name).upload_file('./preprocessed_data.npy', "preprocessed_data.npy") 28 | upload_time = time.time() - upload_start_time 29 | data = json.dumps({"inputs": {"s3_bucket_name": bucket_name, "s3_object_name": "preprocessed_data.npy"}}) 30 | return data, upload_time -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/python_backup/variables.py.sample: -------------------------------------------------------------------------------- 1 | # AWS Sagemaker Region 2 | aws_region = '' 3 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 4 | # ex) inference.com 5 | aws_lambda_address = '' 6 | # aws_lambda bench default address 7 | # https://runtime.sagemaker.{aws_region}.amazonaws.com/endpoints/{aws_sagemaker_endpoint_prefix}-{model_name(with replaced "_" to "-")}-endpoint/invocations 와 같은 형태로 지정됨. 8 | aws_sagemaker_endpoint_prefix = '' 9 | # 추론을 테스트할 모델 이름들 10 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 11 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 12 | num_tasks = [1,10,30] 13 | # CloudWatch Logs Log Group Name 14 | log_group_name = '' 15 | # Bucket for inception_v3 and yolo_v5 (payload size limit issue) 16 | bucket_name = "" -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/run_bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./variables.sh 3 | 4 | # warm bench request lambda instances 5 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --aws_sagemaker_endpoint_prefix "unknown" --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path --tfserving_protocol $tfserving_protocol 6 | sleep 5 7 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --aws_sagemaker_endpoint_prefix "unknown" --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path --tfserving_protocol $tfserving_protocol 8 | sleep 5 9 | for model_name in "${model_names[@]}" 10 | do 11 | for task_num in "${task_nums[@]}" 12 | do 13 | go run run_bench.go --model_name $model_name --task_num $task_num --log_group_name $log_group_name --server_address $server_address --aws_sagemaker_endpoint_prefix $aws_sagemaker_endpoint_prefix --s3_bucket_name $s3_bucket_name --s3_preprocessed_data_key_path $s3_preprocessed_data_key_path --tfserving_protocol $tfserving_protocol 14 | sleep 30 15 | done 16 | done -------------------------------------------------------------------------------- /bench/bench_in_faas/aws_sagemaker/variables.sh.samples: -------------------------------------------------------------------------------- 1 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) inference.com 3 | server_address="" 4 | aws_sagemaker_endpoint_prefix="" 5 | # rest or grpc 6 | tfserving_protocol="" 7 | # 추론을 테스트할 모델 이름들 8 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 9 | # 테스트 추론 횟수들 지정 (각 요쳥 횟수별 병렬로 한번에 처리됩니다.) 10 | task_nums=( 10 20 30 ) 11 | # S3 Bucket Name 12 | s3_bucket_name="" 13 | # Bucket url with preprocessed_data (s3://$s3_bucket_name$s3_folder_name/$model_name.json) 14 | # key path = rest or sagmekaer-grpc 15 | s3_preprocessed_data_key_path="/" 16 | # CloudWatch Logs Log Group Name 17 | log_group_name="" -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/IaC/cloudfunction.tf: -------------------------------------------------------------------------------- 1 | resource "google_cloudfunctions_function" "cloudfunctions_function" { 2 | name = "${var.prefix}-scpc-grpc-bench" 3 | region = var.region 4 | runtime = "python310" 5 | source_archive_bucket = var.bucket_name 6 | source_archive_object = "scpc_grpc_bench.zip" 7 | entry_point = "function_handler" 8 | available_memory_mb = 4096 9 | min_instances = 0 10 | max_instances = 300 11 | timeout = 120 12 | trigger_http = true 13 | https_trigger_security_level = "SECURE_ALWAYS" 14 | environment_variables = { 15 | AWS_ACCESS_KEY_ID = var.aws_access_key 16 | AWS_SECRET_ACCESS_KEY = var.aws_secret_access_key 17 | AWS_REGION = var.aws_region 18 | } 19 | timeouts { 20 | create = "15m" 21 | update = "15m" 22 | } 23 | } 24 | 25 | resource "google_cloudfunctions_function_iam_member" "cloudfunction_noauth" { 26 | region = google_cloudfunctions_function.cloudfunctions_function.region 27 | project = google_cloudfunctions_function.cloudfunctions_function.project 28 | cloud_function = google_cloudfunctions_function.cloudfunctions_function.name 29 | role = "roles/cloudfunctions.invoker" 30 | member = "allUsers" 31 | } 32 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.project 3 | region = var.region 4 | } 5 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloudfunction_url" { 2 | value = google_cloudfunctions_function.cloudfunctions_function.https_trigger_url 3 | } 4 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "region" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "bucket_name" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "aws_access_key" { 22 | type = string 23 | sensitive = true 24 | default = "change here!" 25 | } 26 | 27 | variable "aws_secret_access_key" { 28 | type = string 29 | sensitive = true 30 | default = "change here!" 31 | } 32 | 33 | variable "aws_region" { 34 | type = string 35 | default = "change here!' 36 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/README.md: -------------------------------------------------------------------------------- 1 | This is GCP Functions code to bench GCP Run. -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bucket_name="" 3 | zip -j scpc_grpc_bench.zip ./requirements.txt ./main.py 4 | gsutil cp scpc_grpc_bench.zip gs://$bucket_name 5 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 3 | tensorflow-serving-api==2.11.1 4 | tensorflow==2.11.1 5 | protobuf==3.19.6 -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | from keras.preprocessing.text import Tokenizer 7 | from keras.utils import pad_sequences 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def run_preprocessing(text): 11 | tokenizer = Tokenizer() 12 | tokenizer.fit_on_texts([text]) 13 | input_ids = tokenizer.texts_to_sequences([text]) 14 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 15 | input_masks = [[1] * len(input_ids[0])] 16 | segment_ids = [[0] * len(input_ids[0])] 17 | return input_ids, input_masks, segment_ids 18 | 19 | def create_request_data(): 20 | text = "This is a sample sentence to test the BERT model." 21 | input_ids, input_masks, segment_ids = run_preprocessing(text) 22 | data = predict_pb2.PredictRequest() 23 | data.model_spec.name = 'bert_imdb' 24 | data.model_spec.signature_name = 'serving_default' 25 | data.inputs['input_ids'].CopyFrom(make_tensor_proto(input_ids, shape=[1, len(input_ids[0])])) 26 | data.inputs['input_masks'].CopyFrom(make_tensor_proto(input_masks, shape=[1, len(input_masks[0])])) 27 | data.inputs['segment_ids'].CopyFrom(make_tensor_proto(segment_ids, shape=[1, len(segment_ids[0])])) 28 | json_data = MessageToJson(data) 29 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | from keras.preprocessing.text import Tokenizer 7 | from keras.utils import pad_sequences 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def run_preprocessing(text): 11 | tokenizer = Tokenizer() 12 | tokenizer.fit_on_texts([text]) 13 | bert_input_ids = tokenizer.texts_to_sequences([text]) 14 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 15 | bert_input_masks = [[1] * len(bert_input_ids[0])] 16 | 17 | return bert_input_ids, bert_input_masks 18 | 19 | def create_request_data(): 20 | text = "This is a sample sentence to test the BERT model." 21 | bert_input_ids, bert_input_masks = run_preprocessing(text) 22 | data = predict_pb2.PredictRequest() 23 | data.model_spec.name = 'distilbert_sst2' 24 | data.model_spec.signature_name = 'serving_default' 25 | data.inputs['bert_input_ids'].CopyFrom(make_tensor_proto(bert_input_ids, shape=[1,128])) 26 | data.inputs['bert_input_masks'].CopyFrom(make_tensor_proto(bert_input_masks, shape=[1,128])) 27 | json_data = MessageToJson(data) 28 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for inception) 14 | def run_preprocessing(image_file_path): 15 | img = Image.open(get_file_path(image_file_path)) 16 | img = img.resize((299, 299)) 17 | img_array = np.array(img) 18 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 19 | img_array = img_array.astype(np.float32) 20 | img_array = np.expand_dims(img_array, axis=0) 21 | return img_array 22 | 23 | def create_request_data(): 24 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 25 | data = predict_pb2.PredictRequest() 26 | data.model_spec.name = 'inception_v3' 27 | data.model_spec.signature_name = 'serving_default' 28 | data.inputs['input_3'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 29 | json_data = MessageToJson(data) 30 | return json_data 31 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | 11 | def get_file_path(filename): 12 | return os.path.join(os.path.dirname(__file__), filename) 13 | 14 | # 이미지 로드 및 전처리 (for mobilenet) 15 | def run_preprocessing(image_file_path): 16 | img = Image.open(get_file_path(image_file_path)) 17 | img = img.resize((224, 224)) 18 | img_array = np.array(img) 19 | img_array = img_array.astype('float32') / 255.0 20 | img_array = np.expand_dims(img_array, axis=0) 21 | return img_array 22 | 23 | def create_request_data(): 24 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 25 | data = predict_pb2.PredictRequest() 26 | data.model_spec.name = 'mobilenet_v1' 27 | data.model_spec.signature_name = 'serving_default' 28 | data.inputs['input_1'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 29 | json_data = MessageToJson(data) 30 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for mobilenet) 14 | def run_preprocessing(image_file_path): 15 | img = Image.open(get_file_path(image_file_path)) 16 | img = img.resize((224, 224)) 17 | img_array = np.array(img) 18 | img_array = img_array.astype('float32') / 255.0 19 | img_array = np.expand_dims(img_array, axis=0) 20 | return img_array 21 | 22 | def create_request_data(): 23 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 24 | data = predict_pb2.PredictRequest() 25 | data.model_spec.name = 'mobilenet_v2' 26 | data.model_spec.signature_name = 'serving_default' 27 | data.inputs['input_2'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 28 | json_data = MessageToJson(data) 29 | return json_data 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | import cv2 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for yolo) 14 | def run_preprocessing(image_file_path): 15 | img = cv2.imread(get_file_path(image_file_path)) 16 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 17 | img = cv2.resize(img, (640, 640)) 18 | img = img.astype('float32') / 255.0 19 | img = np.expand_dims(img, axis=0) 20 | return img 21 | 22 | def create_request_data(): 23 | image_file_path = "../../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 24 | data = predict_pb2.PredictRequest() 25 | data.model_spec.name = 'yolo_v5' 26 | data.model_spec.signature_name = 'serving_default' 27 | data.inputs['x'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 28 | json_data = MessageToJson(data) 29 | return json_data 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/grpc/variables.py.sample: -------------------------------------------------------------------------------- 1 | # GCP Run 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) -.cloudfunctions.net/ 3 | # 추론 시, https://-.cloudfunctions.net/과 같은 형식으로 진행된다. 4 | gcp_function_address = '' 5 | # gcp_run prefix 6 | gcp_run_prefix = '' 7 | # gcp_run bench default address ex) $prefix-xxxxxxxxxx.du.a.run.app 8 | gcp_run_default_address = '' 9 | # grpc https options 10 | use_https = "1" 11 | # 추론을 테스트할 모델 이름들 12 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 13 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 14 | num_tasks = [1,10,30] 15 | # CloudWatch Logs Log Group Name 16 | log_group_name = '' -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/IaC/cloudfunction.tf: -------------------------------------------------------------------------------- 1 | resource "google_cloudfunctions_function" "cloudfunctions_function" { 2 | name = "${var.prefix}-scpc-rest-bench" 3 | region = var.region 4 | runtime = "python310" 5 | source_archive_bucket = var.bucket_name 6 | source_archive_object = "scpc_rest_bench.zip" 7 | entry_point = "function_handler" 8 | available_memory_mb = 4096 9 | min_instances = 0 10 | max_instances = 300 11 | timeout = 120 12 | trigger_http = true 13 | https_trigger_security_level = "SECURE_ALWAYS" 14 | environment_variables = { 15 | AWS_ACCESS_KEY_ID = var.aws_access_key 16 | AWS_SECRET_ACCESS_KEY = var.aws_secret_access_key 17 | AWS_REGION = var.aws_region 18 | } 19 | timeouts { 20 | create = "15m" 21 | update = "15m" 22 | } 23 | } 24 | 25 | resource "google_cloudfunctions_function_iam_member" "cloudfunction_noauth" { 26 | region = google_cloudfunctions_function.cloudfunctions_function.region 27 | project = google_cloudfunctions_function.cloudfunctions_function.project 28 | cloud_function = google_cloudfunctions_function.cloudfunctions_function.name 29 | role = "roles/cloudfunctions.invoker" 30 | member = "allUsers" 31 | } 32 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.project 3 | region = var.region 4 | } 5 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloudfunction_url" { 2 | value = google_cloudfunctions_function.cloudfunctions_function.https_trigger_url 3 | } 4 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "region" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "bucket_name" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "aws_access_key" { 22 | type = string 23 | sensitive = true 24 | default = "change here!" 25 | } 26 | 27 | variable "aws_secret_access_key" { 28 | type = string 29 | sensitive = true 30 | default = "change here!" 31 | } 32 | 33 | variable "aws_region" { 34 | type = string 35 | default = "change here!' 36 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bucket_name="" 3 | zip -j scpc_rest_bench.zip ./requirements.txt ./main.py 4 | gsutil cp scpc_rest_bench.zip gs://$bucket_name -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | input_ids = tokenizer.texts_to_sequences([text]) 12 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 13 | input_masks = [[1] * len(input_ids[0])] 14 | segment_ids = [[0] * len(input_ids[0])] 15 | return input_ids, input_masks, segment_ids 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | input_ids, input_masks, segment_ids = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 21 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | bert_input_ids = tokenizer.texts_to_sequences([text]) 12 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 13 | bert_input_masks = [[1] * len(bert_input_ids[0])] 14 | 15 | return bert_input_ids, bert_input_masks 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | bert_input_ids, bert_input_masks = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 21 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for inception) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((299, 299)) 14 | img_array = np.array(img) 15 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 16 | img_array = img_array.astype(np.float32) 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array 19 | 20 | def create_request_data(): 21 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 22 | data = json.dumps({"inputs": { "input_3": run_preprocessing(image_file_path).tolist()}}) 23 | return data 24 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_1": run_preprocessing(image_file_path).tolist()}}) 22 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_2": run_preprocessing(image_file_path).tolist()}}) 22 | return data 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | import cv2 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for yolo) 11 | def run_preprocessing(image_file_path): 12 | img = cv2.imread(get_file_path(image_file_path)) 13 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 14 | img = cv2.resize(img, (640, 640)) 15 | img = img.astype('float32') / 255.0 16 | img = np.expand_dims(img, axis=0) 17 | return img 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 21 | data = json.dumps({"inputs": {"x": run_preprocessing(image_file_path).tolist()}}) 22 | return data 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/gcp_functions_backup/rest/variables.py.sample: -------------------------------------------------------------------------------- 1 | # GCP Run 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) -.cloudfunctions.net/ 3 | # 추론 시, https://-.cloudfunctions.net/과 같은 형식으로 진행된다. 4 | gcp_function_address = '' 5 | # gcp_run prefix 6 | gcp_run_prefix = '' 7 | # gcp_run bench default address ex) $prefix-xxxxxxxxxx.du.a.run.app 8 | gcp_run_default_address = '' 9 | # 추론을 테스트할 모델 이름들 10 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 11 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 12 | num_tasks = [1,10,30] 13 | # CloudWatch Logs Log Group Name 14 | log_group_name = '' -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/IaC/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_route53_zone" "route53_zone" { 2 | name = "${var.route53_domain}." 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | value = aws_apigatewayv2_domain_name.api_domain_name.domain_name 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "route53_domain" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "docker_registry" { 22 | type = string 23 | default = "change here!" 24 | } 25 | 26 | variable "docker_image_tag" { 27 | type = string 28 | default = "change here!" 29 | } 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/build_assets/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !./app.py 3 | !./requirements.txt 4 | !./Dockerfile 5 | !./*.json -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/build_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.10 2 | COPY requirements.txt / 3 | RUN pip3 install --no-cache-dir -r /requirements.txt --target "${LAMBDA_TASK_ROOT}" 4 | COPY app.py ${LAMBDA_TASK_ROOT} 5 | COPY *.json ${LAMBDA_TASK_ROOT} 6 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com/bench-lambda 3 | #you need to create ECR repo and ecr login in instnace 4 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" "distilbert_sst2" ) 5 | for model_name in "${model_names[@]}" 6 | do 7 | if [ ! -e "${model_name}.json" ]; then 8 | curl -o "${model_name}.json" https://ddps-lab-scpc-bucket.s3.ap-northeast-2.amazonaws.com/grpc/$model_name.json 9 | fi 10 | done 11 | 12 | DOCKER_REPOSITORY_NAME="" 13 | 14 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REPOSITORY_NAME:latest -f ./Dockerfile . --build-arg DOCKERIGNORE_FILE=./dockerignore 15 | sudo docker builder prune -f 16 | sudo docker push $DOCKER_REPOSITORY_NAME:latest -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 3 | tensorflow-serving-api==2.11.1 4 | tensorflow==2.11.1 5 | protobuf==3.19.6 -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/go.mod: -------------------------------------------------------------------------------- 1 | module run_bench.go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect 7 | github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect 8 | github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect 9 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect 10 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect 11 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect 12 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect 13 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 // indirect 14 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect 15 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect 16 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect 17 | github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect 18 | github.com/aws/smithy-go v1.13.5 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | from keras.preprocessing.text import Tokenizer 7 | from keras.utils import pad_sequences 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def run_preprocessing(text): 11 | tokenizer = Tokenizer() 12 | tokenizer.fit_on_texts([text]) 13 | input_ids = tokenizer.texts_to_sequences([text]) 14 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 15 | input_masks = [[1] * len(input_ids[0])] 16 | segment_ids = [[0] * len(input_ids[0])] 17 | return input_ids, input_masks, segment_ids 18 | 19 | def create_request_data(): 20 | text = "This is a sample sentence to test the BERT model." 21 | input_ids, input_masks, segment_ids = run_preprocessing(text) 22 | data = predict_pb2.PredictRequest() 23 | data.model_spec.name = 'bert_imdb' 24 | data.model_spec.signature_name = 'serving_default' 25 | data.inputs['input_ids'].CopyFrom(make_tensor_proto(input_ids, shape=[1, len(input_ids[0])])) 26 | data.inputs['input_masks'].CopyFrom(make_tensor_proto(input_masks, shape=[1, len(input_masks[0])])) 27 | data.inputs['segment_ids'].CopyFrom(make_tensor_proto(segment_ids, shape=[1, len(segment_ids[0])])) 28 | json_data = MessageToJson(data) 29 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | from keras.preprocessing.text import Tokenizer 7 | from keras.utils import pad_sequences 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def run_preprocessing(text): 11 | tokenizer = Tokenizer() 12 | tokenizer.fit_on_texts([text]) 13 | bert_input_ids = tokenizer.texts_to_sequences([text]) 14 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 15 | bert_input_masks = [[1] * len(bert_input_ids[0])] 16 | 17 | return bert_input_ids, bert_input_masks 18 | 19 | def create_request_data(): 20 | text = "This is a sample sentence to test the BERT model." 21 | bert_input_ids, bert_input_masks = run_preprocessing(text) 22 | data = predict_pb2.PredictRequest() 23 | data.model_spec.name = 'distilbert_sst2' 24 | data.model_spec.signature_name = 'serving_default' 25 | data.inputs['bert_input_ids'].CopyFrom(make_tensor_proto(bert_input_ids, shape=[1,128])) 26 | data.inputs['bert_input_masks'].CopyFrom(make_tensor_proto(bert_input_masks, shape=[1,128])) 27 | json_data = MessageToJson(data) 28 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for inception) 14 | def run_preprocessing(image_file_path): 15 | img = Image.open(get_file_path(image_file_path)) 16 | img = img.resize((299, 299)) 17 | img_array = np.array(img) 18 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 19 | img_array = img_array.astype(np.float32) 20 | img_array = np.expand_dims(img_array, axis=0) 21 | return img_array 22 | 23 | def create_request_data(): 24 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 25 | data = predict_pb2.PredictRequest() 26 | data.model_spec.name = 'inception_v3' 27 | data.model_spec.signature_name = 'serving_default' 28 | data.inputs['input_3'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 29 | json_data = MessageToJson(data) 30 | return json_data 31 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | 11 | def get_file_path(filename): 12 | return os.path.join(os.path.dirname(__file__), filename) 13 | 14 | # 이미지 로드 및 전처리 (for mobilenet) 15 | def run_preprocessing(image_file_path): 16 | img = Image.open(get_file_path(image_file_path)) 17 | img = img.resize((224, 224)) 18 | img_array = np.array(img) 19 | img_array = img_array.astype('float32') / 255.0 20 | img_array = np.expand_dims(img_array, axis=0) 21 | return img_array 22 | 23 | def create_request_data(): 24 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 25 | data = predict_pb2.PredictRequest() 26 | data.model_spec.name = 'mobilenet_v1' 27 | data.model_spec.signature_name = 'serving_default' 28 | data.inputs['input_1'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 29 | json_data = MessageToJson(data) 30 | return json_data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | from PIL import Image 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for mobilenet) 14 | def run_preprocessing(image_file_path): 15 | img = Image.open(get_file_path(image_file_path)) 16 | img = img.resize((224, 224)) 17 | img_array = np.array(img) 18 | img_array = img_array.astype('float32') / 255.0 19 | img_array = np.expand_dims(img_array, axis=0) 20 | return img_array 21 | 22 | def create_request_data(): 23 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 24 | data = predict_pb2.PredictRequest() 25 | data.model_spec.name = 'mobilenet_v2' 26 | data.model_spec.signature_name = 'serving_default' 27 | data.inputs['input_2'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 28 | json_data = MessageToJson(data) 29 | return json_data 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | from tensorflow_serving.apis import predict_pb2 5 | from tensorflow import make_tensor_proto 6 | import numpy as np 7 | import cv2 8 | from google.protobuf.json_format import MessageToJson 9 | 10 | def get_file_path(filename): 11 | return os.path.join(os.path.dirname(__file__), filename) 12 | 13 | # 이미지 로드 및 전처리 (for yolo) 14 | def run_preprocessing(image_file_path): 15 | img = cv2.imread(get_file_path(image_file_path)) 16 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 17 | img = cv2.resize(img, (640, 640)) 18 | img = img.astype('float32') / 255.0 19 | img = np.expand_dims(img, axis=0) 20 | return img 21 | 22 | def create_request_data(): 23 | image_file_path = "../../../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 24 | data = predict_pb2.PredictRequest() 25 | data.model_spec.name = 'yolo_v5' 26 | data.model_spec.signature_name = 'serving_default' 27 | data.inputs['x'].CopyFrom(make_tensor_proto(run_preprocessing(image_file_path))) 28 | json_data = MessageToJson(data) 29 | return json_data 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/python_backup/variables.py.sample: -------------------------------------------------------------------------------- 1 | # GCP Run 추론 요청 서버의 주소, 모델 이름 설정 2 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 3 | # ex) inference.com 4 | aws_lambda_address = '' 5 | # gcp_run prefix 6 | gcp_run_prefix = '' 7 | # gcp_run bench default address ex) xxxxxxxxxx.du.a.run.app 8 | gcp_run_default_address = '' 9 | # grpc https options 10 | use_https = "1" 11 | # 추론을 테스트할 모델 이름들 12 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 13 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 14 | num_tasks = [1,10,30] 15 | # CloudWatch Logs Log Group Name 16 | log_group_name = '' -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/run_bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./variables.sh 3 | 4 | # warm bench request lambda instances 5 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --gcp_run_prefix "unknown" --gcp_run_default_address "unknown" --use_https $use_https 6 | sleep 5 7 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --gcp_run_prefix "unknown" --gcp_run_default_address "unknown" --use_https $use_https 8 | sleep 5 9 | for model_name in "${model_names[@]}" 10 | do 11 | for task_num in "${task_nums[@]}" 12 | do 13 | go run run_bench.go --model_name $model_name --task_num $task_num --log_group_name $log_group_name --server_address $server_address --gcp_run_prefix $gcp_run_prefix --gcp_run_default_address $gcp_run_default_address --use_https $use_https 14 | sleep 30 15 | done 16 | done -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/grpc/variables.sh.samples: -------------------------------------------------------------------------------- 1 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) inference.com 3 | server_address="" 4 | # gcp_run prefix 5 | gcp_run_prefix="" 6 | # gcp_run bench default address ex) xxxxxxxxxx.du.a.run.app 7 | gcp_run_default_address="" 8 | # grpc https options 9 | use_https="1" 10 | # 추론을 테스트할 모델 이름들 11 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 12 | # 테스트 추론 횟수들 지정 (각 요쳥 횟수별 병렬로 한번에 처리됩니다.) 13 | task_nums=( 10 20 30 ) 14 | # CloudWatch Logs Log Group Name 15 | log_group_name="" -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/IaC/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_route53_zone" "route53_zone" { 2 | name = "${var.route53_domain}." 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/IaC/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/IaC/outputs.tf: -------------------------------------------------------------------------------- 1 | output "api_gateway_url" { 2 | value = aws_apigatewayv2_domain_name.api_domain_name.domain_name 3 | } -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/IaC/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "route53_domain" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "docker_registry" { 22 | type = string 23 | default = "change here!" 24 | } 25 | 26 | variable "docker_image_tag" { 27 | type = string 28 | default = "change here!" 29 | } 30 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/build_assets/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !./app.py 3 | !./requirements.txt 4 | !./Dockerfile 5 | !./*.json -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/build_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.10 2 | COPY requirements.txt / 3 | RUN pip3 install --no-cache-dir -r /requirements.txt --target "${LAMBDA_TASK_ROOT}" 4 | COPY app.py ${LAMBDA_TASK_ROOT} 5 | COPY *.json ${LAMBDA_TASK_ROOT} 6 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/build_assets/build.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com/bench-lambda 3 | #you need to create ECR repo and ecr login in instnace 4 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" "distilbert_sst2" ) 5 | for model_name in "${model_names[@]}" 6 | do 7 | if [ ! -e "${model_name}.json" ]; then 8 | curl -o "${model_name}.json" https://ddps-lab-scpc-bucket.s3.ap-northeast-2.amazonaws.com/rest/$model_name.json 9 | fi 10 | done 11 | 12 | DOCKER_REPOSITORY_NAME="" 13 | 14 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REPOSITORY_NAME:latest -f ./Dockerfile . --build-arg DOCKERIGNORE_FILE=./dockerignore 15 | sudo docker builder prune -f 16 | sudo docker push $DOCKER_REPOSITORY_NAME:latest -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/build_assets/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | boto3 -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/go.mod: -------------------------------------------------------------------------------- 1 | module run_bench.go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect 7 | github.com/aws/aws-sdk-go-v2/config v1.18.27 // indirect 8 | github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect 9 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect 10 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect 11 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect 12 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect 13 | github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.21.2 // indirect 14 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect 15 | github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect 16 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect 17 | github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect 18 | github.com/aws/smithy-go v1.13.5 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | input_ids = tokenizer.texts_to_sequences([text]) 12 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 13 | input_masks = [[1] * len(input_ids[0])] 14 | segment_ids = [[0] * len(input_ids[0])] 15 | return input_ids, input_masks, segment_ids 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | input_ids, input_masks, segment_ids = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 21 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | bert_input_ids = tokenizer.texts_to_sequences([text]) 12 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 13 | bert_input_masks = [[1] * len(bert_input_ids[0])] 14 | 15 | return bert_input_ids, bert_input_masks 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | bert_input_ids, bert_input_masks = run_preprocessing(text) 20 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 21 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for inception) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((299, 299)) 14 | img_array = np.array(img) 15 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 16 | img_array = img_array.astype(np.float32) 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array 19 | 20 | def create_request_data(): 21 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 22 | data = json.dumps({"inputs": { "input_3": run_preprocessing(image_file_path).tolist()}}) 23 | return data 24 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_1": run_preprocessing(image_file_path).tolist()}}) 22 | return data -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = json.dumps({"inputs": { "input_2": run_preprocessing(image_file_path).tolist()}}) 22 | return data 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/preprocess/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | import cv2 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for yolo) 11 | def run_preprocessing(image_file_path): 12 | img = cv2.imread(get_file_path(image_file_path)) 13 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 14 | img = cv2.resize(img, (640, 640)) 15 | img = img.astype('float32') / 255.0 16 | img = np.expand_dims(img, axis=0) 17 | return img 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 21 | data = json.dumps({"inputs": {"x": run_preprocessing(image_file_path).tolist()}}) 22 | return data 23 | -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/python_backup/variables.py.sample: -------------------------------------------------------------------------------- 1 | # GCP Run 추론 요청 서버의 주소, 모델 이름 설정 2 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 3 | # ex) inference.com 4 | aws_lambda_address = '' 5 | # gcp_run prefix 6 | gcp_run_prefix = '' 7 | # gcp_run bench default address ex) xxxxxxxxxx.du.a.run.app 8 | gcp_run_default_address = '' 9 | # 추론을 테스트할 모델 이름들 10 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 11 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 12 | num_tasks = [1,10,30] 13 | # CloudWatch Logs Log Group Name 14 | log_group_name = '' -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/run_bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./variables.sh 3 | 4 | # warm bench request lambda instances 5 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --gcp_run_prefix "unknown" --gcp_run_default_address "unknown" 6 | sleep 5 7 | go run run_bench.go --model_name "unknown" --task_num 100 --log_group_name "unknown" --server_address $server_address --gcp_run_prefix "unknown" --gcp_run_default_address "unknown" 8 | sleep 5 9 | for model_name in "${model_names[@]}" 10 | do 11 | for task_num in "${task_nums[@]}" 12 | do 13 | go run run_bench.go --model_name $model_name --task_num $task_num --log_group_name $log_group_name --server_address $server_address --gcp_run_prefix $gcp_run_prefix --gcp_run_default_address $gcp_run_default_address 14 | sleep 30 15 | done 16 | done -------------------------------------------------------------------------------- /bench/bench_in_faas/gcp_run/rest/variables.sh.samples: -------------------------------------------------------------------------------- 1 | # AWS Lambda 추론 요청 서버의 주소, 모델 이름 설정 2 | # ex) inference.com 3 | server_address="" 4 | # gcp_run prefix 5 | gcp_run_prefix="" 6 | # gcp_run bench default address ex) xxxxxxxxxx.du.a.run.app 7 | gcp_run_default_address="" 8 | # 추론을 테스트할 모델 이름들 9 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 10 | # 테스트 추론 횟수들 지정 (각 요쳥 횟수별 병렬로 한번에 처리됩니다.) 11 | task_nums=( 10 20 30 ) 12 | # CloudWatch Logs Log Group Name 13 | log_group_name="" -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/.gitignore: -------------------------------------------------------------------------------- 1 | *.json -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/bert_imdb.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | input_ids = tokenizer.texts_to_sequences([text]) 12 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 13 | input_masks = [[1] * len(input_ids[0])] 14 | segment_ids = [[0] * len(input_ids[0])] 15 | return input_ids, input_masks, segment_ids 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | input_ids, input_masks, segment_ids = run_preprocessing(text) 20 | data = {"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}} 21 | with open("bert_imdb.json", "w") as file: 22 | json.dump(data, file) 23 | 24 | if __name__ == '__main__': 25 | create_request_data() -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/distilbert_sst2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | import json 7 | 8 | def run_preprocessing(text): 9 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 10 | tokenizer.fit_on_texts([text]) 11 | bert_input_ids = tokenizer.texts_to_sequences([text]) 12 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 13 | bert_input_masks = [[1] * len(bert_input_ids[0])] 14 | 15 | return bert_input_ids, bert_input_masks 16 | 17 | def create_request_data(): 18 | text = "This is a sample sentence to test the BERT model." 19 | bert_input_ids, bert_input_masks = run_preprocessing(text) 20 | data = {"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}} 21 | with open("distilbert_sst2.json", "w") as file: 22 | json.dump(data, file) 23 | 24 | if __name__ == '__main__': 25 | create_request_data() 26 | -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/inception_v3.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for inception) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((299, 299)) 14 | img_array = np.array(img) 15 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 16 | img_array = img_array.astype(np.float32) 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array 19 | 20 | def create_request_data(): 21 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 22 | data = {"inputs": { "input_3": run_preprocessing(image_file_path).tolist()}} 23 | with open("inception_v3.json", "w") as file: 24 | json.dump(data, file) 25 | 26 | if __name__ == '__main__': 27 | create_request_data() 28 | -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/mobilenet_v1.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = {"inputs": { "input_1": run_preprocessing(image_file_path).tolist()}} 22 | with open("mobilenet_v1.json", "w") as file: 23 | json.dump(data, file) 24 | 25 | if __name__ == '__main__': 26 | create_request_data() 27 | -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/mobilenet_v2.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | from PIL import Image 5 | import json 6 | 7 | def get_file_path(filename): 8 | return os.path.join(os.path.dirname(__file__), filename) 9 | 10 | # 이미지 로드 및 전처리 (for mobilenet) 11 | def run_preprocessing(image_file_path): 12 | img = Image.open(get_file_path(image_file_path)) 13 | img = img.resize((224, 224)) 14 | img_array = np.array(img) 15 | img_array = img_array.astype('float32') / 255.0 16 | img_array = np.expand_dims(img_array, axis=0) 17 | return img_array 18 | 19 | def create_request_data(): 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = {"inputs": { "input_2": run_preprocessing(image_file_path).tolist()}} 22 | with open("mobilenet_v2.json", "w") as file: 23 | json.dump(data, file) 24 | 25 | if __name__ == '__main__': 26 | create_request_data() -------------------------------------------------------------------------------- /bench/bench_in_faas/generate_request_data/rest/yolo_v5.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | import numpy as np 4 | import cv2 5 | import json 6 | import boto3 7 | import time 8 | 9 | def get_file_path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | # 이미지 로드 및 전처리 (for yolo) 13 | def run_preprocessing(image_file_path): 14 | img = cv2.imread(get_file_path(image_file_path)) 15 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 16 | img = cv2.resize(img, (640, 640)) 17 | img = img.astype('float32') / 255.0 18 | img = np.expand_dims(img, axis=0) 19 | return img 20 | 21 | def create_request_data(): 22 | image_file_path = "../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 23 | data = {"inputs": {"x": run_preprocessing(image_file_path).tolist()}} 24 | with open("yolo_v5.json", "w") as file: 25 | json.dump(data, file) 26 | 27 | if __name__ == '__main__': 28 | create_request_data() -------------------------------------------------------------------------------- /bench/bench_in_instance/README.md: -------------------------------------------------------------------------------- 1 | - module_gspread를 이용하기 위해서는 module 디렉터리에 Google Spread Sheet 인증 정보를 소유하고 있는 "sheet_credential.json"파일이 있어야 합니다. -------------------------------------------------------------------------------- /bench/bench_in_instance/aws_lambda_variables.py.sample: -------------------------------------------------------------------------------- 1 | # AWS Lambda 모델 서버의 주소, 모델 이름 설정 2 | # AWS Lambda (API Gateway) address default 3 | # ex) inference.com 4 | # 추론 시, https://.inference.com/ 과 같은 형식으로 진행된다. 5 | aws_lambda_default_address = '' 6 | # yolo_v5 추론 시 사용할 S3 Bucket name 7 | bucket_name = '' 8 | # bench 결과를 저장할 google spread sheet id 9 | # spreadsheet안 worksheet는 아래와 같이 저장됩니다. ex) $model_name-$count 10 | spreadsheet_id = '' 11 | # 추론을 테스트할 모델 이름들 12 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 13 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 14 | num_tasks = [1,10,30] 15 | -------------------------------------------------------------------------------- /bench/bench_in_instance/azure_function_variables.py.sample: -------------------------------------------------------------------------------- 1 | # Azure Function모델 서버의 주소 및 포트 번호, 모델 이름 설정 2 | # Azure Function address default (exclude model name) 3 | # ex) -$prefix.azurewebsites.net 4 | # 추론 시, -$prefix.azurewebsites.net 과 같은 형식으로 진행된다. 5 | azure_function_default_address = '' 6 | # blob connection string 7 | blob_connection_string = '' 8 | # blob container name 9 | blob_container_name = '' 10 | # bench 결과를 저장할 google spread sheet id 11 | # spreadsheet안 worksheet는 아래와 같이 저장됩니다. ex) $model_name-$count 12 | spreadsheet_id = '' 13 | # 추론을 테스트할 모델 이름들 14 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 15 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 16 | num_tasks = [1,10,30] 17 | -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_aws_infra/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_ami" "ubuntu_ami" { 2 | most_recent = true 3 | filter { 4 | name = "name" 5 | values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server*"] 6 | } 7 | filter { 8 | name = "virtualization-type" 9 | values = ["hvm"] 10 | } 11 | owners = ["099720109477"] 12 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_aws_infra/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | profile = var.awscli_profile 4 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_aws_infra/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bench-instance-id" { 2 | value = aws_instance.bench_instance.id 3 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_aws_infra/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "awscli_profile" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "keypair" { 17 | type = string 18 | default = "change here!" 19 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_azure_infra/main.tf: -------------------------------------------------------------------------------- 1 | provider "azurerm" { 2 | features {} 3 | } 4 | 5 | data "azurerm_resource_group" "bench-resource-group" { 6 | name = var.resource_group_name 7 | } 8 | 9 | data "azurerm_ssh_public_key" "ssh-key" { 10 | name = var.ssh-keyname 11 | resource_group_name = data.azurerm_resource_group.bench-resource-group.name 12 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_azure_infra/network.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_virtual_network" "bench-network" { 2 | name = "${var.prefix}-scpc-bench-network" 3 | location = data.azurerm_resource_group.bench-resource-group.location 4 | resource_group_name = data.azurerm_resource_group.bench-resource-group.name 5 | address_space = [ "192.168.0.0/16", "2001:db8::/48" ] 6 | } 7 | 8 | resource "azurerm_subnet" "bench-network-subnet" { 9 | name = "${var.prefix}-scpc-bench-network-subnet" 10 | virtual_network_name = azurerm_virtual_network.bench-network.name 11 | resource_group_name = data.azurerm_resource_group.bench-resource-group.name 12 | address_prefixes = [ "192.168.0.0/24", "2001:db8::/64" ] 13 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_azure_infra/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bench-vm-ip" { 2 | value = azurerm_public_ip.bench-vm-public-ipv4.ip_address 3 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_azure_infra/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo apt update 3 | sudo apt install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ 4 | libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ 5 | xz-utils tk-dev liblzma-dev unzip curl docker.io awscli 6 | 7 | sudo echo 'export PYENV_ROOT="/pyenv"' >> /etc/bash.bashrc 8 | sudo echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> /etc/bash.bashrc 9 | sudo echo 'eval "$(pyenv init -)"' >> /etc/bash.bashrc 10 | 11 | sudo git clone https://github.com/pyenv/pyenv.git /pyenv 12 | sudo chmod 777 -R /pyenv 13 | source /etc/bash.bashrc 14 | 15 | pyenv install 3.10.11 16 | sudo chmod 777 -R /pyenv 17 | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_azure_infra/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "location" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "prefix" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "resource_group_name" { 12 | type = string 13 | default = "change here!" 14 | } 15 | 16 | variable "ssh-username" { 17 | type = string 18 | default = "change here!" 19 | } 20 | 21 | variable "ssh-keyname" { 22 | type = string 23 | default = "change here!" 24 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_gcp_infra/README.md: -------------------------------------------------------------------------------- 1 | ### You need to execute shell below at bench_instance after generating infrastructure. 2 | ```bash 3 | pyenv global 3.11.3 4 | cd /serverless-container-performance-comparison/bench 5 | pip3 install -r requirements.txt 6 | ``` -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_gcp_infra/main.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.project 3 | region = var.region 4 | } 5 | -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_gcp_infra/variables.tf.sample: -------------------------------------------------------------------------------- 1 | variable "project" { 2 | type = string 3 | default = "change here!" 4 | } 5 | 6 | variable "region" { 7 | type = string 8 | default = "change here!" 9 | } 10 | 11 | variable "prefix" { 12 | type = string 13 | default = "change here!" 14 | } -------------------------------------------------------------------------------- /bench/bench_in_instance/bench_gcp_infra/vpc.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_network" "bench_network" { 2 | name = "${var.prefix}-serverless-bench-network" 3 | auto_create_subnetworks = false 4 | } 5 | 6 | resource "google_compute_subnetwork" "bench_subnet" { 7 | name = "${var.prefix}-serverless-bench-subnet" 8 | ip_cidr_range = "192.168.0.0/16" 9 | network = google_compute_network.bench_network.id 10 | ipv6_access_type = "EXTERNAL" 11 | stack_type = "IPV4_IPV6" 12 | } 13 | -------------------------------------------------------------------------------- /bench/bench_in_instance/bert_imdb/faas_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from bert_imdb import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_faas 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | text = "This is a sample sentence to test the BERT model." 14 | input_ids, input_masks, segment_ids = preprocessing.run_preprocessing(text) 15 | 16 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_faas.predict(server_address, data)) for _ in range(num_tasks)] 21 | 22 | inference_times = [] 23 | network_latency_times = [] 24 | for future in concurrent.futures.as_completed(futures): 25 | result, thread_elapsed_time = future.result() 26 | inference_times.append(result['inference_time']) 27 | network_latency_times.append(thread_elapsed_time) 28 | 29 | return inference_times, network_latency_times 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/bert_imdb/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | 7 | def run_preprocessing(text): 8 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 9 | tokenizer.fit_on_texts([text]) 10 | input_ids = tokenizer.texts_to_sequences([text]) 11 | input_ids = pad_sequences(input_ids, maxlen=500, padding='post', truncating='post') 12 | input_masks = [[1] * len(input_ids[0])] 13 | segment_ids = [[0] * len(input_ids[0])] 14 | return input_ids, input_masks, segment_ids -------------------------------------------------------------------------------- /bench/bench_in_instance/bert_imdb/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from bert_imdb import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "bert_imdb" 14 | 15 | text = "This is a sample sentence to test the BERT model." 16 | input_ids, input_masks, segment_ids = preprocessing.run_preprocessing(text) 17 | 18 | data = json.dumps({"inputs": { "segment_ids": segment_ids, "input_masks": input_masks, "input_ids": input_ids.tolist()}}) 19 | 20 | # REST 요청 병렬 처리 21 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 22 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 23 | 24 | inference_times_include_network_latency = [] 25 | for future in concurrent.futures.as_completed(futures): 26 | result, thread_elapsed_time = future.result() 27 | inference_times_include_network_latency.append(thread_elapsed_time) 28 | 29 | return inference_times_include_network_latency 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/distilbert_sst2/faas_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from distilbert_sst2 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_faas 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | text = "This is a sample sentence to test the BERT model." 14 | bert_input_ids, bert_input_masks = preprocessing.run_preprocessing(text) 15 | 16 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_faas.predict(server_address, data)) for _ in range(num_tasks)] 21 | 22 | inference_times = [] 23 | network_latency_times = [] 24 | for future in concurrent.futures.as_completed(futures): 25 | result, thread_elapsed_time = future.result() 26 | inference_times.append(result['inference_time']) 27 | network_latency_times.append(thread_elapsed_time) 28 | 29 | return inference_times, network_latency_times 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/distilbert_sst2/grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc_each_session 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | 10 | #preprocessing library 11 | from distilbert_sst2 import preprocessing 12 | 13 | #병렬처리 library 14 | import concurrent.futures 15 | 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "distilbert_sst2" 19 | 20 | text = "This is a sample sentence to test the BERT model." 21 | bert_input_ids, bert_input_masks = preprocessing.run_preprocessing(text) 22 | 23 | # gRPC 요청 생성 24 | request = predict_pb2.PredictRequest() 25 | request.model_spec.name = model_name 26 | request.model_spec.signature_name = 'serving_default' 27 | 28 | request.inputs['bert_input_ids'].CopyFrom(tf.make_tensor_proto(bert_input_ids, shape=[1, 128])) 29 | request.inputs['bert_input_masks'].CopyFrom(tf.make_tensor_proto(bert_input_masks, shape=[1, 128])) 30 | 31 | 32 | # gRPC 요청 병렬 처리 33 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 34 | futures = [executor.submit(lambda: module_grpc_each_session.predict(server_address, use_https, request)) for _ in range(num_tasks)] 35 | 36 | inference_times_include_network_latency = [] 37 | # 결과 출력 38 | for future in concurrent.futures.as_completed(futures): 39 | result, thread_elapsed_time = future.result() 40 | inference_times_include_network_latency.append(thread_elapsed_time) 41 | 42 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/distilbert_sst2/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | from keras.utils import pad_sequences 6 | 7 | def run_preprocessing(text): 8 | tokenizer = tf.keras.preprocessing.text.Tokenizer() 9 | tokenizer.fit_on_texts([text]) 10 | bert_input_ids = tokenizer.texts_to_sequences([text]) 11 | bert_input_ids = pad_sequences(bert_input_ids, maxlen=128, padding='post', truncating='post') 12 | bert_input_masks = [[1] * len(bert_input_ids[0])] 13 | 14 | return bert_input_ids, bert_input_masks -------------------------------------------------------------------------------- /bench/bench_in_instance/distilbert_sst2/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from distilbert_sst2 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "distilbert_sst2" 14 | 15 | text = "This is a sample sentence to test the BERT model." 16 | bert_input_ids, bert_input_masks = preprocessing.run_preprocessing(text) 17 | 18 | data = json.dumps({"inputs": { "bert_input_masks": bert_input_masks, "bert_input_ids": bert_input_ids.tolist()}}) 19 | 20 | # REST 요청 병렬 처리 21 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 22 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 23 | 24 | inference_times_include_network_latency = [] 25 | for future in concurrent.futures.as_completed(futures): 26 | result, thread_elapsed_time = future.result() 27 | inference_times_include_network_latency.append(thread_elapsed_time) 28 | 29 | return inference_times_include_network_latency 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/gcp_functions_variables.py.sample: -------------------------------------------------------------------------------- 1 | # GCP Function 모델 서버의 주소 및 포트 번호, 모델 이름 설정 2 | # GCP Function address default (exclude model name and cloudfunctions.net) 3 | # ex) asia-northeast3-projectname 4 | # 추론 시, -.cloudfunctions.net/function- 과 같은 형식으로 진행된다. 5 | gcp_functions_default_address = '' 6 | # bench 결과를 저장할 google spread sheet id 7 | # spreadsheet안 worksheet는 아래와 같이 저장됩니다. ex) $model_name-$count 8 | spreadsheet_id = '' 9 | # 추론을 테스트할 모델 이름들 10 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5","bert_imdb"] 11 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 12 | num_tasks = [1,10,30] 13 | -------------------------------------------------------------------------------- /bench/bench_in_instance/gcp_run_variables.py.sample: -------------------------------------------------------------------------------- 1 | # Cloud Run 모델 서버의 주소 및 포트 번호, 모델 이름 설정 2 | # Cloud Run address default ex) xxxxxxxxxx.du.a.run.app 3 | gcp_run_default_address = '' 4 | # gRPC에서 https 사용시 1로 설정, http 사용시 0으로 설정 5 | grpc_use_https = 1 6 | # bench 결과를 저장할 google spread sheet id 7 | # spreadsheet안 worksheet는 아래와 같이 저장됩니다. ex) $model_name-$API-$count 8 | spreadsheet_id = '' 9 | # 추론을 테스트할 모델 이름들 10 | model_names = ["mobilenet_v1","mobilenet_v2","inception_v3","yolo_v5"] 11 | # 테스트 추론 횟수들 지정 (병렬로 한번에 처리됩니다.) 12 | num_tasks = [1,10,30] 13 | -------------------------------------------------------------------------------- /bench/bench_in_instance/inception_v3/faas_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from inception_v3 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_faas 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 14 | 15 | data = json.dumps({"inputs": { "input_3" :preprocessing.run_preprocessing(image_file_path).tolist()}}) 16 | 17 | # REST 요청 병렬 처리 18 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 19 | futures = [executor.submit(lambda: module_faas.predict(server_address, data)) for _ in range(num_tasks)] 20 | 21 | inference_times = [] 22 | network_latency_times = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times.append(result['inference_time']) 26 | network_latency_times.append(thread_elapsed_time) 27 | 28 | return inference_times, network_latency_times 29 | 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/inception_v3/grpc_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from inception_v3 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "inception_v3" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | stub = module_grpc.create_grpc_stub(server_address, use_https) 24 | 25 | # gRPC 요청 생성 26 | request = predict_pb2.PredictRequest() 27 | request.model_spec.name = model_name 28 | request.model_spec.signature_name = 'serving_default' 29 | request.inputs['input_3'].CopyFrom(data) 30 | 31 | # gRPC 요청 병렬 처리 32 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 33 | futures = [executor.submit(lambda: module_grpc.predict(stub, request)) for _ in range(num_tasks)] 34 | 35 | inference_times_include_network_latency = [] 36 | # 결과 출력 37 | for future in concurrent.futures.as_completed(futures): 38 | result, thread_elapsed_time = future.result() 39 | inference_times_include_network_latency.append(thread_elapsed_time) 40 | 41 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/inception_v3/grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc_each_session 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from inception_v3 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "inception_v3" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | # gRPC 요청 생성 24 | request = predict_pb2.PredictRequest() 25 | request.model_spec.name = model_name 26 | request.model_spec.signature_name = 'serving_default' 27 | request.inputs['input_3'].CopyFrom(data) 28 | 29 | # gRPC 요청 병렬 처리 30 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 31 | futures = [executor.submit(lambda: module_grpc_each_session.predict(server_address, use_https, request)) for _ in range(num_tasks)] 32 | 33 | inference_times_include_network_latency = [] 34 | # 결과 출력 35 | for future in concurrent.futures.as_completed(futures): 36 | result, thread_elapsed_time = future.result() 37 | inference_times_include_network_latency.append(thread_elapsed_time) 38 | 39 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/inception_v3/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | import numpy as np 6 | from PIL import Image 7 | 8 | def get_file_path(filename): 9 | return os.path.join(os.path.dirname(__file__), filename) 10 | 11 | # 이미지 로드 및 전처리 (for inception) 12 | def run_preprocessing(image_file_path): 13 | img = Image.open(get_file_path(image_file_path)) 14 | img = img.resize((299, 299)) 15 | img_array = np.array(img) 16 | img_array = (img_array - np.mean(img_array)) / np.std(img_array) 17 | img_array = img_array.astype(np.float32) 18 | img_array = np.expand_dims(img_array, axis=0) 19 | return img_array -------------------------------------------------------------------------------- /bench/bench_in_instance/inception_v3/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from inception_v3 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "inception_v3" 14 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 15 | 16 | data = json.dumps({"inputs": { "input_3" :preprocessing.run_preprocessing(image_file_path).tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 21 | 22 | inference_times_include_network_latency = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times_include_network_latency.append(thread_elapsed_time) 26 | 27 | return inference_times_include_network_latency 28 | -------------------------------------------------------------------------------- /bench/bench_in_instance/logs_variables.py.sample: -------------------------------------------------------------------------------- 1 | log_group_name = "" -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v1/faas_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from mobilenet_v1 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_faas 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 14 | 15 | data = json.dumps({"inputs": { "input_1": preprocessing.run_preprocessing(image_file_path).tolist()}}) 16 | 17 | # REST 요청 병렬 처리 18 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 19 | futures = [executor.submit(lambda: module_faas.predict(server_address, data)) for _ in range(num_tasks)] 20 | 21 | inference_times = [] 22 | network_latency_times = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times.append(result['inference_time']) 26 | network_latency_times.append(thread_elapsed_time) 27 | 28 | return inference_times, network_latency_times 29 | -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v1/grpc_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from mobilenet_v1 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "mobilenet_v1" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | stub = module_grpc.create_grpc_stub(server_address, use_https) 24 | 25 | # gRPC 요청 생성 26 | request = predict_pb2.PredictRequest() 27 | request.model_spec.name = model_name 28 | request.model_spec.signature_name = 'serving_default' 29 | request.inputs['input_1'].CopyFrom(data) 30 | 31 | # gRPC 요청 병렬 처리 32 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 33 | futures = [executor.submit(lambda: module_grpc.predict(stub, request)) for _ in range(num_tasks)] 34 | 35 | inference_times_include_network_latency = [] 36 | # 결과 출력 37 | for future in concurrent.futures.as_completed(futures): 38 | result, thread_elapsed_time = future.result() 39 | inference_times_include_network_latency.append(thread_elapsed_time) 40 | 41 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v1/grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc_each_session 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from mobilenet_v1 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "mobilenet_v1" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | # gRPC 요청 생성 24 | request = predict_pb2.PredictRequest() 25 | request.model_spec.name = model_name 26 | request.model_spec.signature_name = 'serving_default' 27 | request.inputs['input_1'].CopyFrom(data) 28 | 29 | # gRPC 요청 병렬 처리 30 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 31 | futures = [executor.submit(lambda: module_grpc_each_session.predict(server_address, use_https, request)) for _ in range(num_tasks)] 32 | 33 | inference_times_include_network_latency = [] 34 | # 결과 출력 35 | for future in concurrent.futures.as_completed(futures): 36 | result, thread_elapsed_time = future.result() 37 | inference_times_include_network_latency.append(thread_elapsed_time) 38 | 39 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v1/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | import numpy as np 6 | from PIL import Image 7 | 8 | def get_file_path(filename): 9 | return os.path.join(os.path.dirname(__file__), filename) 10 | 11 | # 이미지 로드 및 전처리 (for mobilenet) 12 | def run_preprocessing(image_file_path): 13 | img = Image.open(get_file_path(image_file_path)) 14 | img = img.resize((224, 224)) 15 | img_array = np.array(img) 16 | img_array = img_array.astype('float32') / 255.0 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v1/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from mobilenet_v1 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "mobilenet_v1" 14 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 15 | 16 | data = json.dumps({"inputs": { "input_1": preprocessing.run_preprocessing(image_file_path).tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 21 | 22 | inference_times_include_network_latency = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times_include_network_latency.append(thread_elapsed_time) 26 | 27 | return inference_times_include_network_latency 28 | -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v2/faas_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from mobilenet_v2 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_faas 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 14 | 15 | data = json.dumps({"inputs": { "input_2":preprocessing.run_preprocessing(image_file_path).tolist()}}) 16 | 17 | # REST 요청 병렬 처리 18 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 19 | futures = [executor.submit(lambda: module_faas.predict(server_address, data)) for _ in range(num_tasks)] 20 | 21 | inference_times = [] 22 | network_latency_times = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times.append(result['inference_time']) 26 | network_latency_times.append(thread_elapsed_time) 27 | 28 | return inference_times, network_latency_times 29 | 30 | -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v2/grpc_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from mobilenet_v2 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "mobilenet_v2" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | stub = module_grpc.create_grpc_stub(server_address, use_https) 24 | 25 | # gRPC 요청 생성 26 | request = predict_pb2.PredictRequest() 27 | request.model_spec.name = model_name 28 | request.model_spec.signature_name = 'serving_default' 29 | request.inputs['input_2'].CopyFrom(data) 30 | 31 | # gRPC 요청 병렬 처리 32 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 33 | futures = [executor.submit(lambda: module_grpc.predict(stub, request)) for _ in range(num_tasks)] 34 | 35 | inference_times_include_network_latency = [] 36 | # 결과 출력 37 | for future in concurrent.futures.as_completed(futures): 38 | result, thread_elapsed_time = future.result() 39 | inference_times_include_network_latency.append(thread_elapsed_time) 40 | 41 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v2/grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc_each_session 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from mobilenet_v2 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "mobilenet_v2" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | # gRPC 요청 생성 24 | request = predict_pb2.PredictRequest() 25 | request.model_spec.name = model_name 26 | request.model_spec.signature_name = 'serving_default' 27 | request.inputs['input_2'].CopyFrom(data) 28 | 29 | # gRPC 요청 병렬 처리 30 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 31 | futures = [executor.submit(lambda: module_grpc_each_session.predict(server_address, use_https, request)) for _ in range(num_tasks)] 32 | 33 | inference_times_include_network_latency = [] 34 | # 결과 출력 35 | for future in concurrent.futures.as_completed(futures): 36 | result, thread_elapsed_time = future.result() 37 | inference_times_include_network_latency.append(thread_elapsed_time) 38 | 39 | return inference_times_include_network_latency -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v2/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | import numpy as np 6 | from PIL import Image 7 | 8 | def get_file_path(filename): 9 | return os.path.join(os.path.dirname(__file__), filename) 10 | 11 | # 이미지 로드 및 전처리 (for mobilenet) 12 | def run_preprocessing(image_file_path): 13 | img = Image.open(get_file_path(image_file_path)) 14 | img = img.resize((224, 224)) 15 | img_array = np.array(img) 16 | img_array = img_array.astype('float32') / 255.0 17 | img_array = np.expand_dims(img_array, axis=0) 18 | return img_array -------------------------------------------------------------------------------- /bench/bench_in_instance/mobilenet_v2/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from mobilenet_v2 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "mobilenet_v2" 14 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 15 | 16 | data = json.dumps({"inputs": { "input_2":preprocessing.run_preprocessing(image_file_path).tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 21 | 22 | inference_times_include_network_latency = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times_include_network_latency.append(thread_elapsed_time) 26 | 27 | return inference_times_include_network_latency 28 | -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_cw_logs.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import time 4 | 5 | logs_client = boto3.client('logs') 6 | 7 | def create_log_stream(log_group_name, log_stream_name): 8 | logs_client.create_log_stream(logGroupName=log_group_name, logStreamName=log_stream_name) 9 | 10 | def create_log_event(log_group_name, log_stream_name, log_event_index, inference_time, network_latency_time): 11 | log_data = { 12 | 'index_number': log_event_index, 13 | 'inference_time': inference_time, 14 | 'network_latency_time': network_latency_time 15 | } 16 | log_event = { 17 | 'timestamp':int (time.time() * 1000), 18 | 'message': json.dumps(log_data) 19 | } 20 | logs_client.put_log_events(logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=[log_event]) -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_faas.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | 4 | def predict(server_address, data): 5 | headers = {"content-type": "application/json"} 6 | url = server_address 7 | request_time = time.time() 8 | response = requests.post(url, data=data, headers=headers) 9 | response_time = time.time() 10 | elapsed_time = response_time - request_time 11 | result = response.json() 12 | return result, elapsed_time 13 | -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_grpc.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 3 | #gRPC Import 4 | import grpc 5 | from tensorflow_serving.apis import prediction_service_pb2_grpc 6 | 7 | #시간 측정 library 8 | import time 9 | 10 | def create_grpc_stub(server_address, use_https): 11 | # gRPC 채널 생성 12 | if (use_https): 13 | channel = grpc.secure_channel(server_address,grpc.ssl_channel_credentials(),options=[('grpc.max_send_message_length', 50 * 1024 * 1024), ('grpc.max_receive_message_length', 50 * 1024 * 1024)]) 14 | else: 15 | channel = grpc.insecure_channel(server_address,options=[('grpc.max_send_message_length', 50 * 1024 * 1024), ('grpc.max_receive_message_length', 50 * 1024 * 1024)]) 16 | 17 | # gRPC 스텁 생성 18 | stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) 19 | return stub 20 | 21 | def predict(stub, request): 22 | request_time = time.time() 23 | result = stub.Predict(request, timeout=100.0) 24 | response_time = time.time() 25 | elapsed_time = response_time - request_time 26 | return result, elapsed_time -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_grpc_each_session.py: -------------------------------------------------------------------------------- 1 | import os 2 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 3 | #gRPC Import 4 | import grpc 5 | from tensorflow_serving.apis import prediction_service_pb2_grpc 6 | 7 | #시간 측정 library 8 | import time 9 | def predict(server_address, use_https, request): 10 | request_time = time.time() 11 | # gRPC 채널 생성 12 | if (use_https): 13 | channel = grpc.secure_channel(server_address,grpc.ssl_channel_credentials(),options=[('grpc.max_send_message_length', 50 * 1024 * 1024), ('grpc.max_receive_message_length', 50 * 1024 * 1024)]) 14 | else: 15 | channel = grpc.insecure_channel(server_address,options=[('grpc.max_send_message_length', 50 * 1024 * 1024), ('grpc.max_receive_message_length', 50 * 1024 * 1024)]) 16 | 17 | # gRPC 스텁 생성 18 | stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) 19 | result = stub.Predict(request, timeout=100.0) 20 | response_time = time.time() 21 | elapsed_time = response_time - request_time 22 | return result, elapsed_time -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_gspread.py: -------------------------------------------------------------------------------- 1 | import os 2 | import gspread 3 | from oauth2client.service_account import ServiceAccountCredentials 4 | 5 | def get_file_path(filename): 6 | return os.path.join(os.path.dirname(__file__), filename) 7 | 8 | def open_sheet(spreadsheet_key): 9 | scope = [ 10 | 'https://spreadsheets.google.com/feeds' 11 | ] 12 | cred_json_file_name = get_file_path('sheet_credential.json') 13 | credentials = ServiceAccountCredentials.from_json_keyfile_name(cred_json_file_name,scope) 14 | gc = gspread.authorize(credentials) 15 | doc = gc.open_by_key(spreadsheet_key) 16 | return doc -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_rest.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | 4 | def predict(server_address, model_name, data): 5 | headers = {"content-type": "application/json"} 6 | url = server_address + "v1/models/" + model_name + ":predict" 7 | request_time = time.time() 8 | response = requests.post(url, data=data, headers=headers) 9 | response_time = time.time() 10 | elapsed_time = response_time - request_time 11 | result = response.text 12 | return result, elapsed_time 13 | -------------------------------------------------------------------------------- /bench/bench_in_instance/module/module_rest.py.gzip: -------------------------------------------------------------------------------- 1 | import gzip 2 | import base64 3 | import requests 4 | import time 5 | 6 | def predict(server_address, model_name, data): 7 | headers = {"content-type": "application/json"} 8 | url = server_address + "v1/models/" + model_name + ":predict" 9 | request_time = time.time() 10 | compressed_data = gzip.compress(data.encode()) 11 | encoded_data = base64.b64encode(compressed_data).decode() 12 | response = requests.post(url, data=encoded_data, headers=headers) 13 | response_time = time.time() 14 | elapsed_time = response_time - request_time 15 | result = response.text 16 | print(result) 17 | return result, elapsed_time -------------------------------------------------------------------------------- /bench/bench_in_instance/module/put_data_into_sheet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | # google sheet api 관련 library 3 | from module import module_gspread 4 | 5 | def put_data(spreadsheet_id, worksheet_name, array, num_tasks): 6 | spread_doc = module_gspread.open_sheet(spreadsheet_id) 7 | open_worksheet = spread_doc.add_worksheet( 8 | title=worksheet_name, rows=10+num_tasks, cols=10) 9 | open_worksheet.update( 10 | 'A1', "elapsed_inference_time (network latency included)") 11 | open_worksheet.merge_cells("A1:C1") 12 | open_worksheet.update('A2', "Minimum time") 13 | open_worksheet.update( 14 | 'A3', f"=MIN(A6:A{6+num_tasks-1})", value_input_option='USER_ENTERED') 15 | open_worksheet.update('B2', "Maximum time") 16 | open_worksheet.update( 17 | 'B3', f"=MAX(A6:A{6+num_tasks-1})", value_input_option='USER_ENTERED') 18 | open_worksheet.update('C2', "Average time") 19 | open_worksheet.update( 20 | 'C3', f"=AVERAGE(A6:A{6+num_tasks-1})", value_input_option='USER_ENTERED') 21 | open_worksheet.update('A5', "times") 22 | 23 | cell_list = open_worksheet.range(f'A6:A{6+num_tasks-1}') 24 | for i, cell in enumerate(cell_list): 25 | cell.value = array[i] 26 | open_worksheet.update_cells(cell_list) 27 | -------------------------------------------------------------------------------- /bench/bench_in_instance/module/run_faas_bench.py: -------------------------------------------------------------------------------- 1 | from module import put_data_into_sheet 2 | from module import module_cw_logs 3 | import importlib 4 | faas_bench = "" 5 | 6 | def main(model_name, num_tasks, server_address, spreadsheet_id, worksheet_name, service_name, bucket_name="", blob_connection_string='', blob_container_name='', log_group_name='', log_stream_name=''): 7 | global faas_bench 8 | faas_bench = importlib.import_module(f"{model_name}.faas_bench") 9 | if (model_name == "yolo_v5" and service_name == "aws_lambda"): 10 | inference_times, network_latency_times = faas_bench.run_bench(num_tasks, server_address, service_name, bucket_name=bucket_name) 11 | elif (model_name == "yolo_v5" and service_name == "azure_function"): 12 | inference_times, network_latency_times = faas_bench.run_bench(num_tasks, server_address, service_name, blob_connection_string=blob_connection_string, blob_container_name=blob_container_name) 13 | else: 14 | inference_times, network_latency_times = faas_bench.run_bench(num_tasks, server_address) 15 | for i in range(len(inference_times)): 16 | module_cw_logs.create_log_event(log_group_name, log_stream_name, i+1, inference_times[i], network_latency_times[i]) 17 | 18 | #put_data_into_sheet.put_data(spreadsheet_id, worksheet_name, result, num_tasks) -------------------------------------------------------------------------------- /bench/bench_in_instance/module/run_grpc_bench.py: -------------------------------------------------------------------------------- 1 | from module import put_data_into_sheet 2 | import importlib 3 | grpc_bench = "" 4 | 5 | def main(model_name, num_tasks, server_address, use_https, spreadsheet_id, worksheet_name): 6 | global grpc_bench 7 | grpc_bench = importlib.import_module(f"{model_name}.grpc_bench") 8 | result = grpc_bench.run_bench(num_tasks, server_address, use_https) 9 | put_data_into_sheet.put_data(spreadsheet_id, worksheet_name, result, num_tasks) -------------------------------------------------------------------------------- /bench/bench_in_instance/module/run_grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | from module import put_data_into_sheet 2 | import importlib 3 | grpc_bench = "" 4 | 5 | def main(model_name, num_tasks, server_address, use_https, spreadsheet_id, worksheet_name): 6 | global grpc_bench 7 | grpc_bench = importlib.import_module(f"{model_name}.grpc_each_session_bench") 8 | result = grpc_bench.run_bench(num_tasks, server_address, use_https) 9 | put_data_into_sheet.put_data(spreadsheet_id, worksheet_name, result, num_tasks) -------------------------------------------------------------------------------- /bench/bench_in_instance/module/run_rest_bench.py: -------------------------------------------------------------------------------- 1 | from module import put_data_into_sheet 2 | import importlib 3 | rest_bench = "" 4 | 5 | def main(model_name, num_tasks, server_address, spreadsheet_id, worksheet_name): 6 | global rest_bench 7 | rest_bench = importlib.import_module(f"{model_name}.rest_bench") 8 | result = rest_bench.run_bench(num_tasks, server_address) 9 | put_data_into_sheet.put_data(spreadsheet_id, worksheet_name, result, num_tasks) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_aws_lambda_bench.py: -------------------------------------------------------------------------------- 1 | import aws_lambda_variables 2 | import logs_variables 3 | from module import module_cw_logs 4 | from module import run_faas_bench 5 | import time 6 | 7 | def start_bench(model_names, num_tasks, aws_lambda_default_address, spreadsheet_id, bucket_name, log_group_name): 8 | for i, model_name in enumerate(model_names): 9 | for k, num_task in enumerate(num_tasks): 10 | current_timestamp = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime()) 11 | log_stream_name = f"{current_timestamp}-{model_name}-{num_task}tasks" 12 | module_cw_logs.create_log_stream(log_group_name, log_stream_name) 13 | run_faas_bench.main(model_name, 14 | num_task, 15 | f"https://{model_name.replace('_','-')}.{aws_lambda_default_address}/", 16 | spreadsheet_id, 17 | f"{model_name}-{num_task}", 18 | service_name="aws_lambda", 19 | bucket_name=bucket_name, 20 | log_group_name=log_group_name, 21 | log_stream_name=log_stream_name) 22 | time.sleep(5) 23 | 24 | start_bench(aws_lambda_variables.model_names, 25 | aws_lambda_variables.num_tasks, 26 | aws_lambda_variables.aws_lambda_default_address, 27 | aws_lambda_variables.spreadsheet_id, 28 | aws_lambda_variables.bucket_name, 29 | logs_variables.log_group_name) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_azure_function_bench.py: -------------------------------------------------------------------------------- 1 | import azure_function_variables 2 | from module import run_faas_bench 3 | import time 4 | 5 | def start_bench(model_names, num_tasks, azure_function_default_address, spreadsheet_id, blob_connection_string='', blob_container_name=''): 6 | for i, model_name in enumerate(model_names): 7 | for k, num_task in enumerate(num_tasks): 8 | run_faas_bench.main(model_name, 9 | num_task, 10 | f"https://{model_name.replace('_','-')}{azure_function_default_address}/api/predict", 11 | spreadsheet_id, 12 | f"{model_name}-{num_task}", 13 | service_name="azure_function", 14 | blob_connection_string=blob_connection_string, 15 | blob_container_name=blob_container_name) 16 | time.sleep(5) 17 | 18 | start_bench(azure_function_variables.model_names, 19 | azure_function_variables.num_tasks, 20 | azure_function_variables.azure_function_default_address, 21 | azure_function_variables.spreadsheet_id, 22 | azure_function_variables.blob_connection_string, 23 | azure_function_variables.blob_container_name 24 | ) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_gcp_functions_bench.py: -------------------------------------------------------------------------------- 1 | import gcp_functions_variables 2 | from module import run_faas_bench 3 | import time 4 | 5 | def start_bench(model_names, num_tasks, gcp_functions_default_address, spreadsheet_id, bucket_name=""): 6 | for i, model_name in enumerate(model_names): 7 | for k, num_task in enumerate(num_tasks): 8 | run_faas_bench.main(model_name, 9 | num_task, 10 | f"https://{gcp_functions_default_address}.cloudfunctions.net/function-{model_name.replace('_','-')}", 11 | spreadsheet_id, 12 | f"{model_name}-{num_task}", 13 | service_name="gcp_functions") 14 | time.sleep(5) 15 | 16 | start_bench(gcp_functions_variables.model_names, 17 | gcp_functions_variables.num_tasks, 18 | gcp_functions_variables.gcp_functions_default_address, 19 | gcp_functions_variables.spreadsheet_id,) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_gcp_run_grpc_bench.py: -------------------------------------------------------------------------------- 1 | import gcp_run_variables 2 | from module import run_grpc_bench 3 | import time 4 | 5 | def start_bench(model_names, num_tasks, gcp_run_default_address, grpc_use_https, spreadsheet_id): 6 | for i, model_name in enumerate(model_names): 7 | for j, num_task in enumerate(num_tasks): 8 | run_grpc_bench.main(model_name, 9 | num_task, 10 | f"{model_name.replace('_','-')}-grpc-{gcp_run_default_address}", 11 | grpc_use_https, 12 | spreadsheet_id, 13 | f"{model_name}-grpc-{num_task}") 14 | time.sleep(30) 15 | 16 | start_bench(gcp_run_variables.model_names, 17 | gcp_run_variables.num_tasks, 18 | gcp_run_variables.gcp_run_default_address, 19 | gcp_run_variables.grpc_use_https, 20 | gcp_run_variables.spreadsheet_id) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_gcp_run_grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | import gcp_run_variables 2 | from module import run_grpc_each_session_bench 3 | import time 4 | 5 | def start_bench(model_names, num_tasks, gcp_run_default_address, grpc_use_https, spreadsheet_id): 6 | for i, model_name in enumerate(model_names): 7 | for k, num_task in enumerate(num_tasks): 8 | run_grpc_each_session_bench.main(model_name, 9 | num_task, 10 | f"{model_name.replace('_','-')}-grpc-{gcp_run_default_address}", 11 | grpc_use_https, 12 | spreadsheet_id, 13 | f"{model_name}-grpc-each-session-{num_task}") 14 | time.sleep(5) 15 | 16 | start_bench(gcp_run_variables.model_names, 17 | gcp_run_variables.num_tasks, 18 | gcp_run_variables.gcp_run_default_address, 19 | gcp_run_variables.grpc_use_https, 20 | gcp_run_variables.spreadsheet_id) -------------------------------------------------------------------------------- /bench/bench_in_instance/run_gcp_run_rest_bench.py: -------------------------------------------------------------------------------- 1 | import gcp_run_variables 2 | from module import run_rest_bench 3 | import time 4 | 5 | def start_bench(model_names, num_tasks, gcp_run_default_address, spreadsheet_id): 6 | for i, model_name in enumerate(model_names): 7 | for k, num_task in enumerate(num_tasks): 8 | run_rest_bench.main(model_name, 9 | num_task, 10 | f"https://{model_name.replace('_','-')}-rest-{gcp_run_default_address}/", 11 | spreadsheet_id, 12 | f"{model_name}-rest-{num_task}") 13 | time.sleep(30) 14 | 15 | start_bench(gcp_run_variables.model_names, 16 | gcp_run_variables.num_tasks, 17 | gcp_run_variables.gcp_run_default_address, 18 | gcp_run_variables.spreadsheet_id) -------------------------------------------------------------------------------- /bench/bench_in_instance/yolo_v5/grpc_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from yolo_v5 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "yolo_v5" 19 | 20 | image_file_path = "../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | stub = module_grpc.create_grpc_stub(server_address, use_https) 24 | 25 | # gRPC 요청 생성 26 | request = predict_pb2.PredictRequest() 27 | request.model_spec.name = model_name 28 | request.model_spec.signature_name = 'serving_default' 29 | request.inputs['x'].CopyFrom(data) 30 | 31 | # gRPC 요청 병렬 처리 32 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 33 | futures = [executor.submit(lambda: module_grpc.predict(stub, request)) for _ in range(num_tasks)] 34 | 35 | inference_times_include_network_latency = [] 36 | # 결과 출력 37 | for future in concurrent.futures.as_completed(futures): 38 | result, thread_elapsed_time = future.result() 39 | inference_times_include_network_latency.append(thread_elapsed_time) 40 | 41 | return inference_times_include_network_latency 42 | -------------------------------------------------------------------------------- /bench/bench_in_instance/yolo_v5/grpc_each_session_bench.py: -------------------------------------------------------------------------------- 1 | #import grpc module 2 | from module import module_grpc_each_session 3 | from tensorflow_serving.apis import predict_pb2 4 | 5 | #tf log setting 6 | import os 7 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 8 | import tensorflow as tf 9 | import numpy as np 10 | 11 | #preprocessing library 12 | from yolo_v5 import preprocessing 13 | 14 | #병렬처리 library 15 | import concurrent.futures 16 | 17 | def run_bench(num_tasks, server_address, use_https): 18 | model_name = "yolo_v5" 19 | 20 | image_file_path = "../../../dataset/imagenet/imagenet_1000_raw/n01843383_1.JPEG" 21 | data = tf.make_tensor_proto(preprocessing.run_preprocessing(image_file_path)) 22 | 23 | # gRPC 요청 생성 24 | request = predict_pb2.PredictRequest() 25 | request.model_spec.name = model_name 26 | request.model_spec.signature_name = 'serving_default' 27 | request.inputs['x'].CopyFrom(data) 28 | 29 | # gRPC 요청 병렬 처리 30 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 31 | futures = [executor.submit(lambda: module_grpc_each_session.predict(server_address, use_https, request)) for _ in range(num_tasks)] 32 | 33 | inference_times_include_network_latency = [] 34 | # 결과 출력 35 | for future in concurrent.futures.as_completed(futures): 36 | result, thread_elapsed_time = future.result() 37 | inference_times_include_network_latency.append(thread_elapsed_time) 38 | 39 | return inference_times_include_network_latency 40 | -------------------------------------------------------------------------------- /bench/bench_in_instance/yolo_v5/preprocessing.py: -------------------------------------------------------------------------------- 1 | #image 전처리 library 2 | import os 3 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 4 | import tensorflow as tf 5 | import numpy as np 6 | from PIL import Image 7 | import cv2 8 | 9 | def get_file_path(filename): 10 | return os.path.join(os.path.dirname(__file__), filename) 11 | 12 | # 이미지 로드 및 전처리 (for yolo) 13 | def run_preprocessing(image_file_path): 14 | img = cv2.imread(get_file_path(image_file_path)) 15 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 16 | img = cv2.resize(img, (640, 640)) 17 | img = img.astype('float32') / 255.0 18 | img = np.expand_dims(img, axis=0) 19 | 20 | return img 21 | -------------------------------------------------------------------------------- /bench/bench_in_instance/yolo_v5/rest_bench.py: -------------------------------------------------------------------------------- 1 | #preprocessing library 2 | from yolo_v5 import preprocessing 3 | import numpy as np 4 | 5 | #REST 요청 관련 library 6 | from module import module_rest 7 | import json 8 | 9 | #병렬처리 library 10 | import concurrent.futures 11 | 12 | def run_bench(num_tasks, server_address): 13 | model_name = "yolo_v5" 14 | image_file_path = "../../../dataset/coco_2017/coco/images/val2017/000000000139.jpg" 15 | 16 | data = json.dumps({"inputs": {"x": preprocessing.run_preprocessing(image_file_path).tolist()}}) 17 | 18 | # REST 요청 병렬 처리 19 | with concurrent.futures.ThreadPoolExecutor(max_workers=num_tasks) as executor: 20 | futures = [executor.submit(lambda: module_rest.predict(server_address, model_name, data)) for _ in range(num_tasks)] 21 | 22 | inference_times_include_network_latency = [] 23 | for future in concurrent.futures.as_completed(futures): 24 | result, thread_elapsed_time = future.result() 25 | inference_times_include_network_latency.append(thread_elapsed_time) 26 | 27 | return inference_times_include_network_latency 28 | -------------------------------------------------------------------------------- /bench/requirements.txt: -------------------------------------------------------------------------------- 1 | ### Python 3.10.11 2 | #for grpc request 3 | tensorflow==2.11.1 4 | tensorflow-serving-api==2.11.1 5 | protobuf==3.19.6 6 | 7 | 8 | #for image preprocessing 9 | #1) mobilenet / inception 10 | Pillow>=9.5.0 11 | #2) yolo 12 | opencv-python>=4.7.0.72 13 | 14 | #for use google spread sheet 15 | gspread>=5.7.2 16 | oauth2client>=4.1.3 17 | 18 | #for use aws s3 bucket 19 | boto3>=1.26.134 20 | 21 | #for use azure blob container 22 | azure-storage-blob -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/build_assets/aws_lambda_dockerfiles/Dockerfile 3 | !/build_assets/aws_lambda_dockerfiles/requirements.txt -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.10 2 | COPY ./build_assets/aws_lambda_dockerfiles/requirements.txt . 3 | RUN pip3 install --no-cache-dir -r requirements.txt --target "${LAMBDA_TASK_ROOT}" -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/bert_imdb/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/bert_imdb 3 | !/build_assets/aws_lambda_dockerfiles/bert_imdb/app.py 4 | !/build_assets/aws_lambda_dockerfiles/bert_imdb/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/bert_imdb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/bert_imdb ${LAMBDA_TASK_ROOT}/bert_imdb 3 | COPY ./build_assets/aws_lambda_dockerfiles/bert_imdb/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/distilbert_sst2/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/distilbert_sst2 3 | !/build_assets/aws_lambda_dockerfiles/distilbert_sst2/app.py 4 | !/build_assets/aws_lambda_dockerfiles/distilbert_sst2/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/distilbert_sst2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/distilbert_sst2 ${LAMBDA_TASK_ROOT}/distilbert_sst2 3 | COPY ./build_assets/aws_lambda_dockerfiles/distilbert_sst2/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/inception_v3/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/inception_v3 3 | !/build_assets/aws_lambda_dockerfiles/inception_v3/app.py 4 | !/build_assets/aws_lambda_dockerfiles/inception_v3/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/inception_v3/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/inception_v3 ${LAMBDA_TASK_ROOT}/inception_v3 3 | COPY ./build_assets/aws_lambda_dockerfiles/inception_v3/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/mobilenet_v1/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v1 3 | !/build_assets/aws_lambda_dockerfiles/mobilenet_v1/app.py 4 | !/build_assets/aws_lambda_dockerfiles/mobilenet_v1/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/mobilenet_v1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/mobilenet_v1 ${LAMBDA_TASK_ROOT}/mobilenet_v1 3 | COPY ./build_assets/aws_lambda_dockerfiles/mobilenet_v1/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/mobilenet_v2/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v2 3 | !/build_assets/aws_lambda_dockerfiles/mobilenet_v2/app.py 4 | !/build_assets/aws_lambda_dockerfiles/mobilenet_v2/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/mobilenet_v2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/mobilenet_v2 ${LAMBDA_TASK_ROOT}/mobilenet_v2 3 | COPY ./build_assets/aws_lambda_dockerfiles/mobilenet_v2/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/requirements.txt: -------------------------------------------------------------------------------- 1 | #Python 3.10.11 2 | tensorflow-cpu==2.11.1 3 | protobuf==3.19.6 4 | boto3>=1.26.134 -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/yolo_v5/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/yolo_v5 3 | !/build_assets/aws_lambda_dockerfiles/yolo_v5/app.py 4 | !/build_assets/aws_lambda_dockerfiles/yolo_v5/Dockerfile -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/yolo_v5/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aws_lambda_docker_base 2 | COPY ./models/yolo_v5 ${LAMBDA_TASK_ROOT}/yolo_v5 3 | COPY ./build_assets/aws_lambda_dockerfiles/yolo_v5/app.py ${LAMBDA_TASK_ROOT} 4 | CMD [ "app.lambda_handler" ] -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/yolo_v5/app.py.bak: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | import os 4 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 5 | import time 6 | model_load_start_time = time.time() 7 | import tensorflow as tf 8 | model = tf.keras.models.load_model('./yolo_v5') 9 | model_load_end_time = time.time() 10 | 11 | def lambda_handler(event,context): 12 | start_time = time.time() 13 | json_body = json.loads(event['body']) 14 | x = json_body['inputs']['x'] 15 | result = model(np.array(x)) 16 | end_time = time.time() 17 | print(result[0]) 18 | return { 19 | 'statusCode': 200, 20 | 'loading_time': model_load_end_time - model_load_start_time, 21 | 'body': end_time - start_time 22 | } -------------------------------------------------------------------------------- /build_assets/aws_lambda_dockerfiles/yolo_v5/app.py.gzip: -------------------------------------------------------------------------------- 1 | import gzip 2 | import base64 3 | import json 4 | import numpy as np 5 | import os 6 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 7 | import time 8 | model_load_start_time = time.time() 9 | import tensorflow as tf 10 | model = tf.keras.models.load_model('./yolo_v5') 11 | model_load_end_time = time.time() 12 | 13 | def lambda_handler(event,context): 14 | start_time = time.time() 15 | #json_body = json.loads(event['body']) 16 | body = event['body'] 17 | decoded_body = base64.b64decode(body) 18 | uncompressed_body = gzip.decompress(decoded_body) 19 | json_body = json.loads(uncompressed_body) 20 | x = json_body['inputs']['x'] 21 | result = model(np.array(x)) 22 | end_time = time.time() 23 | print(result[0]) 24 | return { 25 | 'statusCode': 200, 26 | 'loading_time': model_load_end_time - model_load_start_time, 27 | 'body': end_time - start_time 28 | } -------------------------------------------------------------------------------- /build_assets/aws_sagemaker_dockerfiles/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM 763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/tensorflow-inference:2.11.1-cpu-py39-ubuntu20.04-sagemaker 2 | COPY ./build_assets/aws_sagemaker_dockerfiles/requirements.txt . 3 | RUN pip3 install --no-cache-dir -r requirements.txt --target "${LAMBDA_TASK_ROOT}" -------------------------------------------------------------------------------- /build_assets/aws_sagemaker_dockerfiles/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow-cpu==2.11.1 -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/build_assets/azure_function_dockerfiles/Dockerfile 3 | !/build_assets/azure_function_dockerfiles/requirements.txt 4 | !/build_assets/azure_function_dockerfiles/host.json 5 | !/build_assets/azure_function_dockerfiles/function.json -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/Dockerfile: -------------------------------------------------------------------------------- 1 | # To enable ssh & remote debugging on app service change the base image to the one below 2 | # FROM mcr.microsoft.com/azure-functions/python:4-python3.10-appservice 3 | FROM mcr.microsoft.com/azure-functions/python:4-python3.10 4 | 5 | ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ 6 | AzureFunctionsJobHost__Logging__Console__IsEnabled=true 7 | 8 | COPY ./build_assets/azure_function_dockerfiles/host.json /home/site/wwwroot/ 9 | COPY ./build_assets/azure_function_dockerfiles/requirements.txt / 10 | 11 | RUN pip --no-cache-dir install -r /requirements.txt -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/bert_imdb/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/bert_imdb 3 | !/build_assets/azure_function_dockerfiles/bert_imdb/Dockerfile 4 | !/build_assets/azure_function_dockerfiles/bert_imdb/__init__.py -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/bert_imdb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azure_function_docker_base 2 | RUN mkdir /home/site/wwwroot/predict 3 | COPY ./build_assets/azure_function_dockerfiles/bert_imdb/__init__.py /home/site/wwwroot/predict/__init__.py 4 | COPY ./build_assets/azure_function_dockerfiles/function.json /home/site/wwwroot/predict/function.json 5 | COPY ./models/bert_imdb /home/site/wwwroot/predict/bert_imdb -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/bert_imdb/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import azure.functions as func 3 | import json 4 | import numpy as np 5 | import os 6 | current_dir = os.path.dirname(os.path.realpath(__file__)) 7 | model_folder = os.path.join(current_dir, "bert_imdb") 8 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 9 | import time 10 | model_load_start_time = time.time() 11 | from tensorflow.keras.models import load_model 12 | model = load_model(model_folder) 13 | model_load_end_time = time.time() 14 | 15 | def main(req: func.HttpRequest) -> func.HttpResponse: 16 | logging.info('Python HTTP trigger function processed a request.') 17 | start_time = time.time() 18 | json_body = req.get_json() 19 | input_ids = np.array(json_body['inputs']['input_ids']) 20 | input_masks = np.array(json_body['inputs']['input_masks']) 21 | segment_ids = np.array(json_body['inputs']['segment_ids']) 22 | result = model.predict([input_masks, input_ids, segment_ids]) 23 | end_time = time.time() 24 | response_data = json.dumps({ 25 | 'loading_time': model_load_end_time - model_load_start_time, 26 | 'inference_time': end_time - start_time, 27 | 'body': result.tolist() 28 | }) 29 | return func.HttpResponse( 30 | status_code=200, 31 | headers={"Content-Type": "application/json"}, 32 | body=response_data 33 | ) -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "__init__.py", 3 | "bindings": [ 4 | { 5 | "authLevel": "Anonymous", 6 | "type": "httpTrigger", 7 | "direction": "in", 8 | "name": "req", 9 | "methods": [ 10 | "get", 11 | "post" 12 | ] 13 | }, 14 | { 15 | "type": "http", 16 | "direction": "out", 17 | "name": "$return" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensionBundle": { 12 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 13 | "version": "[3.*, 4.0.0)" 14 | } 15 | } -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/inception_v3/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/inception_v3 3 | !/build_assets/azure_function_dockerfiles/inception_v3/Dockerfile 4 | !/build_assets/azure_function_dockerfiles/inception_v3/__init__.py -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/inception_v3/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azure_function_docker_base 2 | RUN mkdir /home/site/wwwroot/predict 3 | COPY ./build_assets/azure_function_dockerfiles/inception_v3/__init__.py /home/site/wwwroot/predict/__init__.py 4 | COPY ./build_assets/azure_function_dockerfiles/function.json /home/site/wwwroot/predict/function.json 5 | COPY ./models/inception_v3 /home/site/wwwroot/predict/inception_v3 -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/inception_v3/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import azure.functions as func 3 | import json 4 | import numpy as np 5 | import os 6 | current_dir = os.path.dirname(os.path.realpath(__file__)) 7 | model_folder = os.path.join(current_dir, "inception_v3") 8 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 9 | import time 10 | model_load_start_time = time.time() 11 | from tensorflow.keras.models import load_model 12 | model = load_model(model_folder) 13 | model_load_end_time = time.time() 14 | 15 | def main(req: func.HttpRequest) -> func.HttpResponse: 16 | logging.info('Python HTTP trigger function processed a request.') 17 | start_time = time.time() 18 | json_body = req.get_json() 19 | input_3 = json_body['inputs']['input_3'] 20 | result = model.predict(np.array(input_3)) 21 | end_time = time.time() 22 | response_data = json.dumps({ 23 | 'loading_time': model_load_end_time - model_load_start_time, 24 | 'inference_time': end_time - start_time, 25 | 'body': result.tolist() 26 | }) 27 | return func.HttpResponse( 28 | status_code=200, 29 | headers={"Content-Type": "application/json"}, 30 | body=response_data 31 | ) -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v1/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v1 3 | !/build_assets/azure_function_dockerfiles/mobilenet_v1/Dockerfile 4 | !/build_assets/azure_function_dockerfiles/mobilenet_v1/__init__.py -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azure_function_docker_base 2 | RUN mkdir /home/site/wwwroot/predict 3 | COPY ./build_assets/azure_function_dockerfiles/mobilenet_v1/__init__.py /home/site/wwwroot/predict/__init__.py 4 | COPY ./build_assets/azure_function_dockerfiles/function.json /home/site/wwwroot/predict/function.json 5 | COPY ./models/mobilenet_v1 /home/site/wwwroot/predict/mobilenet_v1 -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v1/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import azure.functions as func 3 | import json 4 | import numpy as np 5 | import os 6 | import sys 7 | current_dir = os.path.dirname(os.path.realpath(__file__)) 8 | model_folder = os.path.join(current_dir, "mobilenet_v1") 9 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 10 | import time 11 | model_load_start_time = time.time() 12 | from tensorflow.keras.models import load_model 13 | model = load_model(model_folder) 14 | model_load_end_time = time.time() 15 | 16 | def main(req: func.HttpRequest) -> func.HttpResponse: 17 | logging.info('Python HTTP trigger function processed a request.') 18 | start_time = time.time() 19 | json_body = req.get_json() 20 | input_1 = json_body['inputs']['input_1'] 21 | result = model.predict(np.array(input_1)) 22 | end_time = time.time() 23 | response_data = json.dumps({ 24 | 'loading_time': model_load_end_time - model_load_start_time, 25 | 'inference_time': end_time - start_time, 26 | 'body': result.tolist() 27 | }) 28 | return func.HttpResponse( 29 | status_code=200, 30 | headers={"Content-Type": "application/json"}, 31 | body=response_data 32 | ) -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v2/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v2 3 | !/build_assets/azure_function_dockerfiles/mobilenet_v2/Dockerfile 4 | !/build_assets/azure_function_dockerfiles/mobilenet_v2/__init__.py -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azure_function_docker_base 2 | RUN mkdir /home/site/wwwroot/predict 3 | COPY ./build_assets/azure_function_dockerfiles/mobilenet_v2/__init__.py /home/site/wwwroot/predict/__init__.py 4 | COPY ./build_assets/azure_function_dockerfiles/function.json /home/site/wwwroot/predict/function.json 5 | COPY ./models/mobilenet_v2 /home/site/wwwroot/predict/mobilenet_v2 -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/mobilenet_v2/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import azure.functions as func 3 | import json 4 | import numpy as np 5 | import os 6 | current_dir = os.path.dirname(os.path.realpath(__file__)) 7 | model_folder = os.path.join(current_dir, "mobilenet_v2") 8 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 9 | import time 10 | model_load_start_time = time.time() 11 | from tensorflow.keras.models import load_model 12 | model = load_model(model_folder) 13 | model_load_end_time = time.time() 14 | 15 | def main(req: func.HttpRequest) -> func.HttpResponse: 16 | logging.info('Python HTTP trigger function processed a request.') 17 | start_time = time.time() 18 | json_body = req.get_json() 19 | input_2 = json_body['inputs']['input_2'] 20 | result = model.predict(np.array(input_2)) 21 | end_time = time.time() 22 | response_data = json.dumps({ 23 | 'loading_time': model_load_end_time - model_load_start_time, 24 | 'inference_time': end_time - start_time, 25 | 'body': result.tolist() 26 | }) 27 | return func.HttpResponse( 28 | status_code=200, 29 | headers={"Content-Type": "application/json"}, 30 | body=response_data 31 | ) -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/requirements.txt: -------------------------------------------------------------------------------- 1 | #Python 3.10.11 2 | azure-functions 3 | azure-storage-blob 4 | tensorflow-cpu==2.11.1 5 | protobuf==3.19.6 -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/yolo_v5/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/yolo_v5 3 | !/build_assets/azure_function_dockerfiles/yolo_v5/Dockerfile 4 | !/build_assets/azure_function_dockerfiles/yolo_v5/__init__.py -------------------------------------------------------------------------------- /build_assets/azure_function_dockerfiles/yolo_v5/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM azure_function_docker_base 2 | RUN mkdir /home/site/wwwroot/predict 3 | COPY ./build_assets/azure_function_dockerfiles/yolo_v5/__init__.py /home/site/wwwroot/predict/__init__.py 4 | COPY ./build_assets/azure_function_dockerfiles/function.json /home/site/wwwroot/predict/function.json 5 | COPY ./models/yolo_v5 /home/site/wwwroot/predict/yolo_v5 -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/bert_imdb/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./bert_imdb') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | input_ids = np.array(json_body['inputs']['input_ids']) 18 | input_masks = np.array(json_body['inputs']['input_masks']) 19 | segment_ids = np.array(json_body['inputs']['segment_ids']) 20 | result = model.predict([input_masks, input_ids, segment_ids]) 21 | end_time = time.time() 22 | response = json.dumps({ 23 | 'loading_time': model_load_end_time - model_load_start_time, 24 | 'inference_time': end_time - start_time, 25 | 'body': result.tolist() 26 | }) 27 | return response, 200, {'Content-Type': 'application/json'} 28 | else: 29 | return json.dumps({ 30 | 'body': "Please send POST request" 31 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/distilbert_sst2/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./distilbert_sst2') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | bert_input_ids = np.array(json_body['inputs']['bert_input_ids']) 18 | bert_input_masks = np.array(json_body['inputs']['bert_input_masks']) 19 | result = model.predict([bert_input_masks, bert_input_ids]) 20 | end_time = time.time() 21 | response = json.dumps({ 22 | 'loading_time': model_load_end_time - model_load_start_time, 23 | 'inference_time': end_time - start_time, 24 | 'body': result.tolist() 25 | }) 26 | return response, 200, {'Content-Type': 'application/json'} 27 | else: 28 | return json.dumps({ 29 | 'body': "Please send POST request" 30 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/inception_v3/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./inception_v3') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | input_3 = json_body['inputs']['input_3'] 18 | result = model.predict(np.array(input_3)) 19 | end_time = time.time() 20 | response = json.dumps({ 21 | 'loading_time': model_load_end_time - model_load_start_time, 22 | 'inference_time': end_time - start_time, 23 | 'body': result.tolist() 24 | }) 25 | return response, 200, {'Content-Type': 'application/json'} 26 | else: 27 | return json.dumps({ 28 | 'body': "Please send POST request" 29 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/mobilenet_v1/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./mobilenet_v1') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | input_1 = json_body['inputs']['input_1'] 18 | result = model.predict(np.array(input_1)) 19 | end_time = time.time() 20 | print("success!") 21 | response = json.dumps({ 22 | 'loading_time': model_load_end_time - model_load_start_time, 23 | 'inference_time': end_time - start_time, 24 | 'body': result.tolist() 25 | }) 26 | return response, 200, {'Content-Type': 'application/json'} 27 | else: 28 | return json.dumps({ 29 | 'body': "Please send POST request" 30 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/mobilenet_v2/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./mobilenet_v2') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | input_2 = json_body['inputs']['input_2'] 18 | result = model.predict(np.array(input_2)) 19 | end_time = time.time() 20 | response = json.dumps({ 21 | 'loading_time': model_load_end_time - model_load_start_time, 22 | 'inference_time': end_time - start_time, 23 | 'body': result.tolist() 24 | }) 25 | return response, 200, {'Content-Type': 'application/json'} 26 | else: 27 | return json.dumps({ 28 | 'body': "Please send POST request" 29 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/requirements.txt: -------------------------------------------------------------------------------- 1 | #Python 3.10.11 2 | tensorflow-cpu==2.11.1 3 | protobuf==3.19.6 -------------------------------------------------------------------------------- /build_assets/gcp_functions_codes/yolo_v5/main.py: -------------------------------------------------------------------------------- 1 | import functions_framework 2 | import json 3 | import numpy as np 4 | import os 5 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 6 | import time 7 | model_load_start_time = time.time() 8 | from tensorflow.keras.models import load_model 9 | model = load_model('./yolo_v5') 10 | model_load_end_time = time.time() 11 | 12 | @functions_framework.http 13 | def predict(request): 14 | if request.method == 'POST': 15 | start_time = time.time() 16 | json_body = request.get_json(silent=True) 17 | x = json_body['inputs']['x'] 18 | result = model(np.array(x)) 19 | end_time = time.time() 20 | response = json.dumps({ 21 | 'loading_time': model_load_end_time - model_load_start_time, 22 | 'inference_time': end_time - start_time, 23 | 'body': result.tolist() 24 | }) 25 | return response, 200, {'Content-Type': 'application/json'} 26 | else: 27 | return json.dumps({ 28 | 'body': "Please send POST request" 29 | }), 403, {'Content-Type': 'application/json'} -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/bert_imdb/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/bert_imdb -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/bert_imdb/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/bert_imdb /models/bert_imdb/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=bert_imdb","--model_base_path=/models/bert_imdb","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/bert_imdb/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/bert_imdb /models/bert_imdb/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=bert_imdb","--model_base_path=/models/bert_imdb","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/distilbert_sst2/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/distilbert_sst2 -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/distilbert_sst2/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/distilbert_sst2 /models/distilbert_sst2/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=distilbert_sst2","--model_base_path=/models/distilbert_sst2","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/distilbert_sst2/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/distilbert_sst2 /models/distilbert_sst2/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=distilbert_sst2","--model_base_path=/models/distilbert_sst2","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/inception_v3/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/inception_v3 -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/inception_v3/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/inception_v3 /models/inception_v3/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=inception_v3","--model_base_path=/models/inception_v3","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/inception_v3/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/inception_v3 /models/inception_v3/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=inception_v3","--model_base_path=/models/inception_v3","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v1/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v1 -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v1/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/mobilenet_v1 /models/mobilenet_v1/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=mobilenet_v1","--model_base_path=/models/mobilenet_v1","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v1/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/mobilenet_v1 /models/mobilenet_v1/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=mobilenet_v1","--model_base_path=/models/mobilenet_v1","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v2/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v2 -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v2/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/mobilenet_v2 /models/mobilenet_v2/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=mobilenet_v2","--model_base_path=/models/mobilenet_v2","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/mobilenet_v2/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/mobilenet_v2 /models/mobilenet_v2/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=mobilenet_v2","--model_base_path=/models/mobilenet_v2","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/yolo_v5/.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !/models/yolo_v5 -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/yolo_v5/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/yolo_v5 /models/yolo_v5/1/ 4 | 5 | EXPOSE 8500 6 | 7 | CMD ["--port=8500","--model_name=yolo_v5","--model_base_path=/models/yolo_v5","--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_dockerfiles/yolo_v5/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM tensorflow/serving:2.11.1 2 | 3 | COPY ./models/yolo_v5 /models/yolo_v5/1/ 4 | 5 | EXPOSE 8501 6 | 7 | CMD ["--rest_api_port=8501","--model_name=yolo_v5","--model_base_path=/models/yolo_v5","--rest_api_num_threads=1000"] 8 | 9 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/build_assets/gcp_run_python_dockerfiles/Dockerfile.grpc 3 | !/build_assets/gcp_run_python_dockerfiles/requirements.grpc.txt -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/.dockerignore.grpc.monitoring: -------------------------------------------------------------------------------- 1 | * 2 | !/build_assets/gcp_run_python_dockerfiles/Dockerfile.grpc 3 | !/build_assets/gcp_run_python_dockerfiles/requirements.grpc.monitoring.txt 4 | !/build_assets/gcp_run_python_dockerfiles/grpc_start.sh 5 | !/build_assets/gcp_run_python_dockerfiles/push_metrics.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/build_assets/gcp_run_python_dockerfiles/Dockerfile.rest 3 | !/build_assets/gcp_run_python_dockerfiles/requirements.rest.txt -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM python:3.10.11 2 | 3 | WORKDIR /app 4 | 5 | COPY ./build_assets/gcp_run_python_dockerfiles/requirements.grpc.txt /app 6 | 7 | RUN pip install --no-cache-dir -r requirements.grpc.txt -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM python:3.10.11 2 | 3 | WORKDIR /app 4 | 5 | COPY ./build_assets/gcp_run_python_dockerfiles/requirements.grpc.monitoring.txt /app 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/grpc_start.sh /app 8 | 9 | COPY ./build_assets/gcp_run_python_dockerfiles/push_metrics.py /app 10 | 11 | RUN pip install --no-cache-dir -r requirements.grpc.monitoring.txt 12 | 13 | RUN chmod 755 ./grpc_start.sh -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM python:3.10.11 2 | 3 | WORKDIR /app 4 | 5 | COPY ./build_assets/gcp_run_python_dockerfiles/requirements.rest.txt /app 6 | 7 | RUN pip install --no-cache-dir -r requirements.rest.txt -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/bert_imdb/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/models/bert_imdb 3 | !/build_assets/gcp_run_python_dockerfiles/bert_imdb/grpc_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/bert_imdb/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/models/bert_imdb 3 | !/build_assets/gcp_run_python_dockerfiles/bert_imdb/rest_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/bert_imdb/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/bert_imdb /app/bert_imdb 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/bert_imdb/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["python", "grpc_main.py"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/bert_imdb/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/bert_imdb /app/bert_imdb 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/bert_imdb/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["./grpc_start.sh"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/bert_imdb/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_rest_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/bert_imdb /app/bert_imdb 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/bert_imdb/rest_main.py /app 8 | 9 | EXPOSE 8501 10 | 11 | CMD ["uvicorn", "rest_main:app", "--port", "8501", "--host", "0.0.0.0"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/grpc_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python3 grpc_main.py & 3 | python3 push_metrics.py & 4 | wait -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/inception_v3/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/models/inception_v3 3 | !/build_assets/gcp_run_python_dockerfiles/inception_v3/grpc_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/inception_v3/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/models/inception_v3 3 | !/build_assets/gcp_run_python_dockerfiles/inception_v3/rest_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/inception_v3/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/inception_v3 /app/inception_v3 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/inception_v3/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["python", "grpc_main.py"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/inception_v3/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/inception_v3 /app/inception_v3 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/inception_v3/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["./grpc_start.sh"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/inception_v3/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_rest_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/inception_v3 /app/inception_v3 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/inception_v3/rest_main.py /app 8 | 9 | EXPOSE 8501 10 | 11 | CMD ["uvicorn", "rest_main:app", "--port", "8501", "--host", "0.0.0.0"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v1/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v1 3 | !/build_assets/gcp_run_python_dockerfiles/mobilenet_v1/grpc_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v1/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v1 3 | !/build_assets/gcp_run_python_dockerfiles/mobilenet_v1/rest_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v1/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v1 /app/mobilenet_v1 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v1/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["python", "grpc_main.py"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v1/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v1 /app/mobilenet_v1 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v1/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["./grpc_start.sh"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v1/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_rest_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v1 /app/mobilenet_v1 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v1/rest_main.py /app 8 | 9 | EXPOSE 8501 10 | 11 | CMD ["uvicorn", "rest_main:app", "--port", "8501", "--host", "0.0.0.0"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v2/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v2 3 | !/build_assets/gcp_run_python_dockerfiles/mobilenet_v2/grpc_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v2/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/models/mobilenet_v2 3 | !/build_assets/gcp_run_python_dockerfiles/mobilenet_v2/rest_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v2/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v2 /app/mobilenet_v2 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v2/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["python", "grpc_main.py"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v2/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v2 /app/mobilenet_v2 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v2/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["./grpc_start.sh"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/mobilenet_v2/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_rest_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/mobilenet_v2 /app/mobilenet_v2 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/mobilenet_v2/rest_main.py /app 8 | 9 | EXPOSE 8501 10 | 11 | CMD ["uvicorn", "rest_main:app", "--port", "8501", "--host", "0.0.0.0"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/requirements.grpc.monitoring.txt: -------------------------------------------------------------------------------- 1 | tensorflow==2.11.1 2 | tensorflow-serving-api==2.11.1 3 | protobuf==3.19.6 4 | grpcio==1.54.2 5 | requests 6 | prometheus-client -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/requirements.grpc.txt: -------------------------------------------------------------------------------- 1 | tensorflow==2.11.1 2 | tensorflow-serving-api==2.11.1 3 | protobuf==3.19.6 4 | grpcio==1.54.2 5 | requests -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/requirements.rest.txt: -------------------------------------------------------------------------------- 1 | tensorflow-cpu==2.11.1 2 | fastapi==0.95.2 3 | uvicorn[standard]==0.22.0 4 | requests -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/yolo_v5/.dockerignore.grpc: -------------------------------------------------------------------------------- 1 | * 2 | !/models/yolo_v5 3 | !/build_assets/gcp_run_python_dockerfiles/yolo_v5/grpc_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/yolo_v5/.dockerignore.rest: -------------------------------------------------------------------------------- 1 | * 2 | !/models/yolo_v5 3 | !/build_assets/gcp_run_python_dockerfiles/yolo_v5/rest_main.py -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/yolo_v5/Dockerfile.grpc: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/yolo_v5 /app/yolo_v5 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/yolo_v5/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["python", "grpc_main.py"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/yolo_v5/Dockerfile.grpc.monitoring: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_grpc_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/yolo_v5 /app/yolo_v5 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/yolo_v5/grpc_main.py /app 8 | 9 | EXPOSE 8500 10 | 11 | CMD ["./grpc_start.sh"] 12 | 13 | -------------------------------------------------------------------------------- /build_assets/gcp_run_python_dockerfiles/yolo_v5/Dockerfile.rest: -------------------------------------------------------------------------------- 1 | FROM gcp_run_python_rest_docker_base 2 | 3 | WORKDIR /app 4 | 5 | COPY ./models/yolo_v5 /app/yolo_v5 6 | 7 | COPY ./build_assets/gcp_run_python_dockerfiles/yolo_v5/rest_main.py /app 8 | 9 | EXPOSE 8501 10 | 11 | CMD ["uvicorn", "rest_main:app", "--port", "8501", "--host", "0.0.0.0"] 12 | 13 | -------------------------------------------------------------------------------- /build_aws_lambda_docker_image.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com 3 | #you need to create ECR repo and ecr login in instnace 4 | DEFAULT_DOCKER_REPOSITORY_NAME="" 5 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 6 | 7 | sudo DOCKER_BUILDKIT=1 docker build -t aws_lambda_docker_base -f ./build_assets/aws_lambda_dockerfiles/Dockerfile . --build-arg DOCKERIGNORE_FILE=./build_assets/aws_lambda_dockerfiles/.dockerignore 8 | for model_name in "${model_names[@]}" 9 | do 10 | sudo DOCKER_BUILDKIT=1 docker build -t $DEFAULT_DOCKER_REPOSITORY_NAME/aws-lambda-$model_name:latest -f ./build_assets/aws_lambda_dockerfiles/$model_name/Dockerfile . --build-arg DOCKERIGNORE_FILE=./build_assets/aws_lambda_dockerfiles/$model_name/.dockerignore 11 | sudo docker builder prune -f 12 | done 13 | 14 | for model_name in "${model_names[@]}" 15 | do 16 | sudo docker push $DEFAULT_DOCKER_REPOSITORY_NAME/aws-lambda-$model_name:latest 17 | done -------------------------------------------------------------------------------- /build_aws_sagemaker_docker_image.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxxxxx.dkr.ecr.ap-northeast-2.amazonaws.com 3 | DOCKER_REGISTRY="" 4 | 5 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/aws-sagemaker-prebuilt-image:latest -f ./build_assets/aws_sagemaker_dockerfiles/Dockerfile . 6 | sudo docker push $DOCKER_REGISTRY/aws-sagemaker-prebuilt-image:latest -------------------------------------------------------------------------------- /build_azure_function_docker_image.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #ex) xxxxxxxx.azurecr.io 3 | #### you need login to acr (sudo az acr login) 4 | DOCKER_REGISTRY="" 5 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 6 | 7 | sudo DOCKER_BUILDKIT=1 docker build -t azure_function_docker_base -f ./build_assets/azure_function_dockerfiles/Dockerfile . --build-arg DOCKERIGNORE_FILE=./build_assets/azure_function_dockerfiles/.dockerignore 8 | for model_name in "${model_names[@]}" 9 | do 10 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/azure-function-$model_name:latest -f ./build_assets/azure_function_dockerfiles/$model_name/Dockerfile . --build-arg DOCKERIGNORE_FILE=./build_assets/azure_function_dockerfiles/$model_name/.dockerignore 11 | sudo docker builder prune -f 12 | done 13 | 14 | for model_name in "${model_names[@]}" 15 | do 16 | sudo docker push $DOCKER_REGISTRY/azure-function-$model_name:latest 17 | done -------------------------------------------------------------------------------- /build_gcp_run_docker_image.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #you need to login Artifact Registry (docker login) 3 | DOCKER_REGISTRY="" 4 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" "bert_imdb" ) 5 | APIS=( "grpc" "rest" ) 6 | 7 | for model_name in "${model_names[@]}" 8 | do 9 | for API in "${APIS[@]}" 10 | do 11 | sudo DOCKER_BUILDKIT=1 docker build -t $DOCKER_REGISTRY/gcp-run-$model_name-$API:latest -f ./build_assets/gcp_run_dockerfiles/$model_name/Dockerfile.$API . --build-arg DOCKERIGNORE_FILE=./build_assets/gcp_run_dockerfiles/$model_name/.dockerignore 12 | sudo docker builder prune -f 13 | done 14 | done 15 | 16 | for model_name in "${model_names[@]}" 17 | do 18 | for API in "${APIS[@]}" 19 | do 20 | sudo docker push $DOCKER_REGISTRY/gcp-run-$model_name-$API:latest 21 | done 22 | done -------------------------------------------------------------------------------- /create_gcp_functions_code_zip.sh.sample: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bucket_name="" 3 | model_names=( "mobilenet_v1" "mobilenet_v2" "inception_v3" "yolo_v5" ) 4 | 5 | for model_name in "${model_names[@]}" 6 | do 7 | cd ./models 8 | zip -r $model_name.zip $model_name 9 | cd .. 10 | mv ./models/$model_name.zip . 11 | zip -j $model_name.zip ./build_assets/gcp_functions_codes/requirements.txt ./build_assets/gcp_functions_codes/$model_name/main.py 12 | gsutil cp $model_name.zip gs://$bucket_name 13 | done -------------------------------------------------------------------------------- /dataset/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### Dataset 다운로드 3 | 4 | ``` 5 | pip3 install -r requirements.txt 6 | chmod +x dataset_download.sh 7 | ./dataset_download.sh 8 | ``` 9 | -------------------------------------------------------------------------------- /dataset/dataset_download.sh: -------------------------------------------------------------------------------- 1 | # #!/bin/bash 2 | 3 | #image classification image dataset (tfrecord imagenet) 4 | mkdir imagenet 5 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/imagenet/imagenet_1000 6 | mv imagenet_1000 ./imagenet 7 | 8 | #image classification image dataset (raw imagenet) 9 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/imagenet/imagenet_metadata.txt 10 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/imagenet/imagenet_1000_raw.zip 11 | mv imagenet_metadata.txt ./imagenet 12 | unzip -q imagenet_1000_raw.zip -d ./imagenet && rm imagenet_1000_raw.zip 13 | 14 | ###object detection image dataset(coco_2017) 15 | mkdir coco_2017 16 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/coco_2017/val_dataset.py 17 | python3 val_dataset.py 18 | mv val_dataset.py ./coco_2017 19 | unzip -q coco2017val.zip -d ./coco_2017 && rm coco2017val.zip -------------------------------------------------------------------------------- /dataset/requirements.txt: -------------------------------------------------------------------------------- 1 | ### 2 | wget>=3.2 -------------------------------------------------------------------------------- /models/README.md: -------------------------------------------------------------------------------- 1 | ### Dataset 다운로드 2 | 3 | ``` 4 | chmod +x dataset_download.sh 5 | ./dataset_download.sh 6 | ``` 7 | ### Model 다운로드 8 | 9 | ``` 10 | chmod +x model_download.sh 11 | ./model_download.sh 12 | ``` 13 | -------------------------------------------------------------------------------- /models/model_download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #image classification TF/FP32 model (mobilenet v1, mobilenet v2, inception v3) 3 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/model/mobilenet_v1/mobilenet_v1.zip 4 | unzip -q mobilenet_v1.zip && rm mobilenet_v1.zip 5 | 6 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/model/mobilenet_v2/mobilenet_v2.zip 7 | unzip -q mobilenet_v2.zip && rm mobilenet_v2.zip 8 | 9 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/model/inception_v3/inception_v3.zip 10 | unzip -q inception_v3.zip && rm inception_v3.zip 11 | 12 | #object detection TF/FP32 model (yolo v5) 13 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/model/yolo_v5/yolo_v5.zip 14 | unzip -q yolo_v5.zip && rm yolo_v5.zip 15 | mv yolov5/yolov5s_saved_model yolo_v5 16 | rm -rf yolov5 17 | 18 | #bert imdb model 19 | curl -O https://edge-inference.s3.us-west-2.amazonaws.com/NLP/bert_imdb.zip 20 | unzip -q bert_imdb.zip && rm bert_imdb.zip 21 | 22 | #distilbert sst2 model 23 | # curl -O https://edge-inference.s3.us-west-2.amazonaws.com/NLP/distilbert_sst2.zip 24 | # unzip -q distilbert_sst2.zip && rm distilbert_sst2.zip 25 | --------------------------------------------------------------------------------