├── .idea ├── .gitignore ├── Yet-Another-Serverless-Benchmark.iml ├── deployment.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── sshConfigs.xml ├── vcs.xml └── webServers.xml ├── README.md ├── automated_test_tool ├── cpu │ └── docker_monitor │ │ ├── .idea │ │ ├── .gitignore │ │ ├── deployment.xml │ │ ├── docker_monitor.iml │ │ ├── modules.xml │ │ ├── sshConfigs.xml │ │ └── webServers.xml │ │ ├── go.mod │ │ ├── m2.go │ │ └── util │ │ ├── RandString.go │ │ ├── execCmd.go │ │ └── sendhttp.go └── throughput │ ├── invoke_mr.sh │ └── testnp.lua ├── finra ├── openfaas │ ├── OpenFaaS.py │ ├── Workflow.py │ ├── deploy_finra.sh │ ├── lastpx.yml │ ├── lastpx │ │ ├── __init__.py │ │ ├── handler.py │ │ ├── portfolios.json │ │ ├── requirements.txt │ │ └── util.py │ ├── margin-balance.yml │ ├── margin-balance │ │ ├── __init__.py │ │ ├── handler.py │ │ ├── marginBalance.json │ │ ├── portfolios.json │ │ ├── requirements.txt │ │ └── util.py │ ├── marketdata.yml │ ├── marketdata │ │ ├── __init__.py │ │ ├── base.py │ │ ├── handler.py │ │ ├── requirements.txt │ │ ├── shared.py │ │ └── utils.py │ ├── rm_finra.sh │ ├── side.yml │ ├── side │ │ ├── __init__.py │ │ ├── handler.py │ │ ├── portfolios.json │ │ ├── requirements.txt │ │ └── util.py │ ├── trddate.yml │ ├── trddate │ │ ├── __init__.py │ │ ├── handler.py │ │ ├── portfolios.json │ │ ├── requirements.txt │ │ └── util.py │ ├── volume.yml │ ├── volume │ │ ├── __init__.py │ │ ├── handler.py │ │ ├── portfolios.json │ │ ├── requirements.txt │ │ └── util.py │ ├── yfinance.yml │ └── yfinance │ │ ├── __init__.py │ │ ├── handler.py │ │ └── requirements.txt ├── ray │ ├── Workflow.py │ ├── deploy.py │ ├── fi │ │ ├── main.py │ │ ├── model │ │ │ └── post.py │ │ ├── service │ │ │ ├── lastpx.py │ │ │ ├── marginBalance.py │ │ │ ├── market │ │ │ │ ├── base.py │ │ │ │ ├── shared.py │ │ │ │ └── utils.py │ │ │ ├── marketdata.py │ │ │ ├── message.py │ │ │ ├── side.py │ │ │ ├── test.py │ │ │ ├── trddate.py │ │ │ ├── volume.py │ │ │ └── yfinance.py │ │ └── service_bak │ │ │ ├── lastpx.py │ │ │ ├── marginBalance.py │ │ │ ├── marketdata.py │ │ │ ├── message.py │ │ │ ├── side.py │ │ │ ├── test.py │ │ │ ├── trddate.py │ │ │ ├── volume.py │ │ │ └── yfinance.py │ └── req.py └── ray_stateful │ ├── Workflow.py │ ├── deploy.py │ ├── fi │ ├── main.py │ ├── model │ │ └── post.py │ ├── service │ │ ├── lastpx.py │ │ ├── marginBalance.py │ │ ├── market │ │ │ ├── 1 │ │ │ ├── base.py │ │ │ ├── shared.py │ │ │ └── utils.py │ │ ├── marketdata.py │ │ ├── message.py │ │ ├── side.py │ │ ├── test.py │ │ ├── trddate.py │ │ ├── volume.py │ │ └── yfinance.py │ └── service_bak │ │ ├── lastpx.py │ │ ├── marginBalance.py │ │ ├── marketdata.py │ │ ├── message.py │ │ ├── side.py │ │ ├── test.py │ │ ├── trddate.py │ │ ├── volume.py │ │ └── yfinance.py │ └── req.py ├── jhdaslkd ├── mapreduce ├── openfaas │ ├── README.md │ └── wc │ │ ├── Workflow.py │ │ ├── clean.py │ │ ├── client.py │ │ ├── functions │ │ ├── template │ │ │ ├── python3-flask-debian │ │ │ │ ├── Dockerfile │ │ │ │ ├── function │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── handler.py │ │ │ │ │ └── requirements.txt │ │ │ │ ├── index.py │ │ │ │ ├── requirements.txt │ │ │ │ └── template.yml │ │ │ └── python3-flask │ │ │ │ ├── .Dockerfile.swp │ │ │ │ ├── Dockerfile │ │ │ │ ├── function │ │ │ │ ├── __init__.py │ │ │ │ ├── handler.py │ │ │ │ └── requirements.txt │ │ │ │ ├── index.py │ │ │ │ ├── requirements.txt │ │ │ │ └── template.yml │ │ ├── wc-mapper.yml │ │ ├── wc-mapper │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── wc-reducer.yml │ │ └── wc-reducer │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── gen_data │ │ ├── Makefile │ │ ├── README │ │ ├── createBK.py │ │ ├── gen_random_text │ │ ├── gen_random_text.cpp │ │ ├── gen_text_data.sh │ │ ├── lda_wiki1w │ │ │ ├── final.beta │ │ │ ├── final.other │ │ │ └── lda_wiki1w.voca │ │ ├── removeBK.py │ │ ├── showBK.py │ │ └── upload.py │ │ └── yaml │ │ ├── minio.yaml │ │ └── redis.yaml ├── ray │ ├── Workflow1.py │ ├── deploy.py │ ├── mapreduce.yaml │ ├── req.py │ └── wc │ │ ├── main.py │ │ ├── model │ │ └── post.py │ │ └── service │ │ ├── map.py │ │ └── reduce.py └── ray_stateful │ ├── Workflow1.py │ ├── deploy.py │ ├── mapreduce.yaml │ ├── req.py │ ├── test.py │ ├── test1.py │ └── wc │ ├── main.py │ ├── model │ └── post.py │ └── service │ ├── map.py │ └── reduce.py ├── media_service ├── openfaas │ └── mr │ │ ├── OpenFaaS.py │ │ ├── Workflow.py │ │ ├── functions │ │ ├── compose-review.yml │ │ ├── compose-review │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── deploy_mr.sh │ │ ├── describe.sh │ │ ├── mr-compose-and-upload.yml │ │ ├── mr-compose-and-upload │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── mr-db-op.yml │ │ ├── mr-db-op │ │ │ ├── __init__.py │ │ │ ├── datasets │ │ │ │ ├── casts.json │ │ │ │ └── movies.json │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── mr-upload-text.yml │ │ ├── mr-upload-text │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── mr-upload-unique-id.yml │ │ ├── mr-upload-unique-id │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── rm_mr.sh │ │ ├── store-review.yml │ │ ├── store-review │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── upload-movie-id.yml │ │ ├── upload-movie-id │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── upload-movie-review.yml │ │ ├── upload-movie-review │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── upload-user-id.yml │ │ ├── upload-user-id │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── upload-user-review.yml │ │ └── upload-user-review │ │ │ ├── __init__.py │ │ │ ├── handler.py │ │ │ └── requirements.txt │ │ ├── movie_titles.csv │ │ └── yaml │ │ ├── namespaces.yml │ │ └── yaml-db │ │ ├── cast-info-mongodb.yaml │ │ ├── movie-id-memcached.yaml │ │ ├── movie-id-mongodb.yaml │ │ ├── movie-info-mongodb.yaml │ │ ├── movie-review-mongodb.yaml │ │ ├── plot-mongodb.yaml │ │ ├── review-storage-mongodb.yaml │ │ ├── user-memcached.yaml │ │ ├── user-mongodb.yaml │ │ └── user-review-mongodb.yaml ├── ray │ ├── Workflow.py │ ├── deploy.py │ ├── movie_titles.csv │ ├── ms │ │ ├── main.py │ │ ├── model │ │ │ └── post.py │ │ ├── requirements │ │ │ └── requirements.txt │ │ └── service │ │ │ ├── compose_review.py │ │ │ ├── lastpx.py │ │ │ ├── mr_compose_and_upload.py │ │ │ ├── mr_upload_text.py │ │ │ ├── mr_upload_unique_id.py │ │ │ ├── store_review.py │ │ │ ├── upload_movie_id.py │ │ │ ├── upload_movie_review.py │ │ │ ├── upload_user_id.py │ │ │ └── upload_user_review.py │ └── req.py └── ray_stateful │ ├── Workflow.py │ ├── deploy.py │ ├── media.yaml │ ├── movie_titles.csv │ ├── ms │ ├── main.py │ ├── model │ │ └── post.py │ ├── movie_titles.csv │ ├── requirements │ │ └── requirements.txt │ └── service │ │ ├── compose_review.py │ │ ├── mr_compose_and_upload.py │ │ ├── mr_upload_text.py │ │ ├── mr_upload_unique_id.py │ │ ├── store_review.py │ │ ├── upload_movie_id.py │ │ ├── upload_movie_review.py │ │ ├── upload_user_id.py │ │ └── upload_user_review.py │ └── req.py └── solver ├── openfaas ├── .gitignore ├── arithmetic-arranger.yml ├── arithmetic-arranger │ ├── __init__.py │ ├── handler.py │ └── requirements.txt ├── curve-fitting.yml ├── curve-fitting │ ├── __init__.py │ ├── handler.py │ └── requirements.txt ├── eigen.yml ├── eigen │ ├── __init__.py │ ├── handler.py │ └── requirements.txt ├── linear-programming.yml ├── linear-programming │ ├── __init__.py │ ├── handler.py │ └── requirements.txt ├── prob-calculator.yml ├── prob-calculator │ ├── __init__.py │ ├── handler.py │ └── requirements.txt ├── readme.md ├── time-calculator.yml └── time-calculator │ ├── __init__.py │ ├── handler.py │ └── requirements.txt └── ray_stateful ├── Workflow.py ├── deploy.py ├── req.py ├── so ├── main.py ├── model │ └── post.py ├── requirements │ └── requirements.txt └── service │ ├── arithmetic_arranger.py │ ├── curve_fitting.py │ ├── eigen.py │ ├── linear_programming.py │ ├── prob_calculator.py │ └── time_calculator.py └── solver.yaml /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/Yet-Another-Serverless-Benchmark.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 15 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/sshConfigs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/webServers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 13 | 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Yet-Another-Serverless-Benchmark -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 15 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/docker_monitor.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/sshConfigs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/.idea/webServers.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 13 | 14 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/go.mod: -------------------------------------------------------------------------------- 1 | module docker_monitor 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/util/RandString.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | func RandString(len int) string { 9 | /* 10 | rand.Seed: 11 | 还函数是用来创建随机数的种子,如果不执行该步骤创建的随机数是一样的，因为默认Go会使用一个固定常量值来作为随机种子。 12 | 13 | time.Now().UnixNano(): 14 | 当前操作系统时间的毫秒值 15 | */ 16 | rand.Seed(time.Now().UnixNano()) 17 | /* 18 | 生成一个随机chuan 19 | */ 20 | var str string 21 | for i := 0; i < len; i++ { 22 | a := rand.Intn(26) 23 | X := 'A'+a 24 | str+=string(X) 25 | } 26 | return str 27 | } 28 | 29 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/util/execCmd.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os/exec" 5 | ) 6 | 7 | func ExecCmd(cmd string) (string,error) { 8 | output, err := exec.Command("bash", "-c", cmd).Output() 9 | result := string(output) 10 | return result,err 11 | } 12 | -------------------------------------------------------------------------------- /automated_test_tool/cpu/docker_monitor/util/sendhttp.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | func HttpPost(requestUrl string, requestId string, reqBody interface{}, customHeaders map[string]string) { 12 | byteData, err := json.Marshal(reqBody) 13 | if err != nil { 14 | fmt.Errorf("err:%v",err) 15 | return 16 | } 17 | body := bytes.NewReader(byteData) 18 | //创建post请求 19 | req, err := http.NewRequest("POST", requestUrl, body) 20 | if err != nil { 21 | fmt.Errorf("request uri failed:%v", err) 22 | return 23 | } 24 | for key, value := range customHeaders { 25 | req.Header.Set(key, value) 26 | } 27 | 28 | req.Header.Set("Content-Type", "application/json;charset=UTF-8") 29 | req.Header.Set("Connection", "close") 30 | if requestId != "" { 31 | req.Header.Set("RequestId", requestId) 32 | } else { 33 | requestId = customHeaders["X-Request-Id"] 34 | } 35 | 36 | headers, _ := json.Marshal(req.Header) 37 | fmt.Printf("request url:%s, headers:%s, data:%s", requestUrl, headers, byteData) 38 | //生成client 39 | client := &http.Client{} 40 | resp := &http.Response{} 41 | for i := 0; ; i++ { 42 | //发送请求 43 | resp, err = client.Do(req) 44 | if err != nil { 45 | fmt.Errorf("request url:%s, body:%s failed.Error msg:%s.", requestUrl, byteData, err) 46 | if i < 3 && resp != nil && resp.StatusCode == http.StatusInternalServerError { 47 | time.Sleep(100 * time.Millisecond) 48 | fmt.Printf("server return 500, try again:%d", i) 49 | continue 50 | } 51 | return 52 | } 53 | resp.Body.Close() 54 | fmt.Printf("requestId:%s response status code:%d", requestId, resp.StatusCode) 55 | return 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /automated_test_tool/throughput/invoke_mr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WRK_BIN=wrk2 4 | WRK_SCRIPT=./testnp.lua 5 | URL=http://localhost:5000 6 | 7 | # runs a benchmark for 3 seconds, using 5 threads, keeping 20 HTTP connections open 8 | # and a constant throughput of 20 requests per second (total, across all connections combined). 9 | #$WRK_BIN -d60s -c16 -t8 -R160 --u_latency -s $WRK_SCRIPT $URL 10 | $WRK_BIN -d30s -c20 -t10 -R50 --latency -s $WRK_SCRIPT $URL 11 | -------------------------------------------------------------------------------- /automated_test_tool/throughput/testnp.lua: -------------------------------------------------------------------------------- 1 | 2 | local counter = 1 3 | local threads = {} 4 | local interval = 60000 5 | 6 | function setup(thread) 7 | thread:set("id", counter) 8 | table.insert(threads, thread) 9 | counter = counter + 1 10 | end 11 | 12 | function init(args) 13 | cnt = 1 14 | math.randomseed(id) 15 | end 16 | 17 | function request() 18 | 19 | 20 | cnt = cnt + 1 21 | local method = "GET" 22 | local path = "/hi" 23 | local headers = {} 24 | headers["Content-Type"] = "Content-Type: application/json" 25 | local body = "{}" 26 | 27 | return wrk.format(method, path, headers, body) 28 | 29 | end 30 | 31 | -------------------------------------------------------------------------------- /finra/openfaas/OpenFaaS.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import threading 3 | 4 | import json 5 | import time 6 | 7 | margin_balance_url = "http://127.0.0.1:31119/function/margin-balance" 8 | 9 | funcs = ["marketdata", "lastpx", "side", "trddate", "volume"] 10 | 11 | urls = [f"http://127.0.0.1:31119/function/{func}" for func in funcs] 12 | 13 | req = '{"body":{ "portfolioType":"S&P", "portfolio":"1234"}}' 14 | 15 | def post(url, data, l): 16 | res = requests.post(url, data).text 17 | l.append(res) 18 | 19 | def workflow(): 20 | start = time.time() 21 | 22 | parallel_res = [] 23 | threads = [] 24 | 25 | for url in urls: 26 | t = threading.Thread(target=post, args=(url, req, parallel_res)) 27 | threads.append(t) 28 | 29 | for t in threads: 30 | t.start() 31 | 32 | for t in threads: 33 | t.join() 34 | 35 | res = requests.post(margin_balance_url, data=json.dumps(parallel_res)).text 36 | 37 | return int((time.time()-start)*1000) 38 | print("This request uses %d so" % int((time.time()-start)*1000)) 39 | 40 | if __name__ == '__main__': 41 | lats = [] 42 | for i in range(11): 43 | lat = workflow() 44 | lats.append(lat) 45 | 46 | # print(lats) 47 | print(sum(lats[1:])/(len(lats)-1)) 48 | -------------------------------------------------------------------------------- /finra/openfaas/Workflow.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import threading 3 | 4 | import json 5 | import time 6 | from flask import Flask 7 | 8 | app = Flask(__name__) 9 | 10 | margin_balance_url = "http://127.0.0.1:31119/function/margin-balance" 11 | 12 | funcs = ["marketdata", "lastpx", "side", "trddate", "volume"] 13 | 14 | urls = [f"http://127.0.0.1:31119/function/{func}" for func in funcs] 15 | 16 | req = '{"body":{ "portfolioType":"S&P", "portfolio":"1234"}}' 17 | 18 | 19 | def post(url, data, l): 20 | res = requests.post(url, data).text 21 | l.append(res) 22 | 23 | 24 | def workflow(): 25 | start = time.time() 26 | 27 | parallel_res = [] 28 | threads = [] 29 | 30 | for url in urls: 31 | t = threading.Thread(target=post, args=(url, req, parallel_res)) 32 | threads.append(t) 33 | 34 | for t in threads: 35 | t.start() 36 | 37 | for t in threads: 38 | t.join() 39 | 40 | res = requests.post(margin_balance_url, data=json.dumps(parallel_res)).text 41 | # print("This request uses %d so" % int((time.time() - start) * 1000)) 42 | return int((time.time() - start) * 1000) 43 | 44 | 45 | 46 | 47 | @app.route('/hi') 48 | def index(): 49 | lats = [] 50 | for i in range(11): 51 | lat = workflow() 52 | lats.append(lat) 53 | 54 | # print(lats) 55 | # print(sum(lats[1:]) / (len(lats) - 1)) 56 | print("This request uses %d so" %(sum(lats[1:]) / (len(lats) - 1))) 57 | return { 58 | "msg": "success", 59 | "data": "welcome" 60 | } 61 | 62 | 63 | if __name__ == "__main__": 64 | app.run() -------------------------------------------------------------------------------- /finra/openfaas/deploy_finra.sh: -------------------------------------------------------------------------------- 1 | funcs=("marketdata" "lastpx" "side" "trddate" "volume" "margin-balance" "yfinance") 2 | for ((i=0;i<7;i++)) 3 | do 4 | faashwh-cli build -f ${funcs[$i]}.yml 5 | done 6 | 7 | for ((i=0;i<7;i++)) 8 | do 9 | faashwh-cli deploy -f ${funcs[$i]}.yml 10 | done 11 | -------------------------------------------------------------------------------- /finra/openfaas/lastpx.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | lastpx: 7 | lang: python3-flask 8 | handler: ./lastpx 9 | image: lastpx:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /finra/openfaas/lastpx/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/lastpx/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/lastpx/handler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | # from function.util import * 4 | 5 | def timestamp(response, event, startTime, endTime, externalServicesTime): 6 | stampBegin = 1000*time.time() 7 | prior = event['duration'] if 'duration' in event else 0 8 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 9 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 10 | response['duration'] = prior + endTime - startTime 11 | response['workflowEndTime'] = endTime 12 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 13 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 14 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 15 | response['memsetTime'] = priorMemSet 16 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 17 | return response 18 | 19 | def handle(req): 20 | """handle a request to the function 21 | Args: 22 | req (str): request body 23 | """ 24 | event = json.loads(req) 25 | 26 | startTime = 1000*time.time() 27 | 28 | portfolio = event['body']['portfolio'] 29 | portfolios = json.loads(open('/home/app/function/portfolios.json', 'r').read()) 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | px = str(trade['LastPx']) 36 | if '.' in px: 37 | a,b = px.split('.') 38 | if not ((len(a) == 3 and len(b) == 6) or 39 | (len(a) == 4 and len(b) == 5) or 40 | (len(a) == 5 and len(b) == 4) or 41 | (len(a) == 6 and len(b) == 3)): 42 | print('{}: {}v{}'.format(px, len(a), len(b))) 43 | valid = False 44 | break 45 | 46 | response = {'statusCode': 200, 'body': {'valid':valid, 'portfolio': portfolio}} 47 | endTime = 1000*time.time() 48 | return json.dumps(timestamp(response, event,startTime, endTime, 0)) 49 | -------------------------------------------------------------------------------- /finra/openfaas/lastpx/portfolios.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : [ 3 | { 4 | "Security": "GOOG", 5 | "LastQty" : 10, 6 | "LastPx" : 1363.85123, 7 | "Side" : 1, 8 | "TrdSubType" : 0, 9 | "TradeDate" : "200507" 10 | }, 11 | { 12 | "Security": "MSFT", 13 | "LastQty" : 20, 14 | "LastPx" : 183.851234, 15 | "Side" : 1, 16 | "TrdSubType" : 0, 17 | "TradeDate" : "200507" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /finra/openfaas/lastpx/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/lastpx/requirements.txt -------------------------------------------------------------------------------- /finra/openfaas/margin-balance.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | margin-balance: 7 | lang: python3-flask 8 | handler: ./margin-balance 9 | image: margin-balance:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | constraints: 13 | - "kubernetes.io/hostname=kube-node-7" 14 | -------------------------------------------------------------------------------- /finra/openfaas/margin-balance/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/margin-balance/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/margin-balance/marginBalance.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : 4500 3 | } 4 | -------------------------------------------------------------------------------- /finra/openfaas/margin-balance/portfolios.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : [ 3 | { 4 | "Security": "GOOG", 5 | "LastQty" : 10, 6 | "LastPx" : 1363.85123, 7 | "Side" : 1, 8 | "TrdSubType" : 0, 9 | "TradeDate" : "200507" 10 | }, 11 | { 12 | "Security": "MSFT", 13 | "LastQty" : 20, 14 | "LastPx" : 183.851234, 15 | "Side" : 1, 16 | "TrdSubType" : 0, 17 | "TradeDate" : "200507" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /finra/openfaas/margin-balance/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/margin-balance/requirements.txt -------------------------------------------------------------------------------- /finra/openfaas/marketdata.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | marketdata: 7 | lang: python3-flask-debian 8 | handler: ./marketdata 9 | image: marketdata:latest 10 | environment: 11 | write_timeout: 1m 12 | read_timeout: 1m 13 | exec_timeout: 1m 14 | handler_wait_duration: 1m 15 | labels: 16 | "com.openfaas.scale.min": "1" 17 | "com.openfaas.scale.max": "1" 18 | constraints: 19 | - "kubernetes.io/hostname=kube-node-7" 20 | -------------------------------------------------------------------------------- /finra/openfaas/marketdata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/marketdata/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/marketdata/handler.py: -------------------------------------------------------------------------------- 1 | from function import base 2 | import time 3 | import json 4 | 5 | 6 | def handle(req): 7 | """handle a request to the function 8 | Args: 9 | req (str): request body 10 | """ 11 | event = json.loads(req) 12 | 13 | startTime = time.time() 14 | externalServicesTime = [] 15 | portfolioType = event['body']['portfolioType'] 16 | 17 | tickersForPortfolioTypes = {'S&P': ['GOOG', 'AMZN', 'MSFT']} 18 | tickers = tickersForPortfolioTypes[portfolioType] 19 | 20 | prices = {} 21 | for ticker in tickers: 22 | tickerObj = base.Ticker(ticker) 23 | #Get last closing price 24 | tickTime = time.time() 25 | data = tickerObj.history(period="1") 26 | externalServicesTime.append(time.time() - tickTime) 27 | price = data['Close'].unique()[0] 28 | prices[ticker] = price 29 | 30 | # prices = {'GOOG': 1732.38, 'AMZN': 3185.27, 'MSFT': 221.02} 31 | 32 | endTime = time.time() 33 | 34 | response = {'time': {'start': startTime, 'end': endTime, 'externalServicesTime': externalServicesTime}, 'body': {'marketData':prices}} 35 | 36 | return json.dumps(response) 37 | 38 | return req 39 | -------------------------------------------------------------------------------- /finra/openfaas/marketdata/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | pandas 3 | numpy 4 | -------------------------------------------------------------------------------- /finra/openfaas/marketdata/shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # yfinance - market data downloader 5 | # https://github.com/ranaroussi/yfinance 6 | # 7 | # Copyright 2017-2019 Ran Aroussi 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | _DFS = {} 23 | _PROGRESS_BAR = None 24 | _ERRORS = {} 25 | _ISINS = {} 26 | -------------------------------------------------------------------------------- /finra/openfaas/rm_finra.sh: -------------------------------------------------------------------------------- 1 | funcs=("marketdata-y" "lastpx" "side" "trddate" "volume" "margin-balance" "yfinance") 2 | for ((i=0;i<7;i++)) 3 | do 4 | faas-cli remove -f ${funcs[$i]}.yml 5 | done 6 | -------------------------------------------------------------------------------- /finra/openfaas/side.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | side: 7 | lang: python3-flask 8 | handler: ./side 9 | image: side:latest 10 | labels: 11 | "com.openfaas.scale.min": "1" 12 | "com.openfaas.scale.max": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /finra/openfaas/side/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/side/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/side/handler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | # from function.util import * 4 | 5 | def timestamp(response, event, startTime, endTime, externalServicesTime): 6 | stampBegin = 1000*time.time() 7 | prior = event['duration'] if 'duration' in event else 0 8 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 9 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 10 | response['duration'] = prior + endTime - startTime 11 | response['workflowEndTime'] = endTime 12 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 13 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 14 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 15 | response['memsetTime'] = priorMemSet 16 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 17 | return response 18 | 19 | 20 | def handle(req): 21 | """handle a request to the function 22 | Args: 23 | req (str): request body 24 | """ 25 | event = json.loads(req) 26 | 27 | startTime = 1000*time.time() 28 | 29 | portfolio = event['body']['portfolio'] 30 | portfolios = json.loads(open('/home/app/function/portfolios.json', 'r').read()) 31 | data = portfolios[portfolio] 32 | 33 | valid = True 34 | 35 | for trade in data: 36 | side = trade['Side'] 37 | # Tag ID: 552, Tag Name: Side, Valid values: 1,2,8 38 | if not (side == 1 or side == 2 or side == 8): 39 | valid = False 40 | break 41 | 42 | response = {'statusCode': 200, 'body': {'valid':valid, 'portfolio': portfolio}} 43 | endTime = 1000*time.time() 44 | return json.dumps(timestamp(response, event, startTime, endTime, 0)) 45 | -------------------------------------------------------------------------------- /finra/openfaas/side/portfolios.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : [ 3 | { 4 | "Security": "GOOG", 5 | "LastQty" : 10, 6 | "LastPx" : 1363.85123, 7 | "Side" : 1, 8 | "TrdSubType" : 0, 9 | "TradeDate" : "200507" 10 | }, 11 | { 12 | "Security": "MSFT", 13 | "LastQty" : 20, 14 | "LastPx" : 183.851234, 15 | "Side" : 1, 16 | "TrdSubType" : 0, 17 | "TradeDate" : "200507" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /finra/openfaas/side/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/side/requirements.txt -------------------------------------------------------------------------------- /finra/openfaas/trddate.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | trddate: 7 | lang: python3-flask 8 | handler: ./trddate 9 | image: trddate:latest 10 | labels: 11 | "com.openfaas.scale.min": "1" 12 | "com.openfaas.scale.max": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /finra/openfaas/trddate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/trddate/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/trddate/handler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | # from function.util import * 5 | 6 | def timestamp(response, event, startTime, endTime, externalServicesTime): 7 | stampBegin = 1000*time.time() 8 | prior = event['duration'] if 'duration' in event else 0 9 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 10 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 11 | response['duration'] = prior + endTime - startTime 12 | response['workflowEndTime'] = endTime 13 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 14 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 15 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 16 | response['memsetTime'] = priorMemSet 17 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 18 | return response 19 | 20 | def handle(req): 21 | """handle a request to the function 22 | Args: 23 | req (str): request body 24 | """ 25 | event = json.loads(req) 26 | 27 | startTime = 1000*time.time() 28 | 29 | portfolio = event['body']['portfolio'] 30 | portfolios = json.loads(open('/home/app/function/portfolios.json', 'r').read()) 31 | data = portfolios[portfolio] 32 | 33 | valid = True 34 | 35 | for trade in data: 36 | trddate = trade['TradeDate'] 37 | # Tag ID: 75, Tag Name: TradeDate, Format: YYMMDD 38 | if len(trddate) == 6: 39 | try: 40 | datetime.datetime(int(trddate[0:2]), int(trddate[2:4]), int(trddate[4:6])) 41 | except ValueError: 42 | valid = False 43 | break 44 | else: 45 | valid = False 46 | break 47 | 48 | response = {'statusCode': 200, 'body': {'valid':valid, 'portfolio': portfolio}} 49 | endTime = 1000*time.time() 50 | return json.dumps(timestamp(response, event, startTime, endTime, 0)) 51 | -------------------------------------------------------------------------------- /finra/openfaas/trddate/portfolios.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : [ 3 | { 4 | "Security": "GOOG", 5 | "LastQty" : 10, 6 | "LastPx" : 1363.85123, 7 | "Side" : 1, 8 | "TrdSubType" : 0, 9 | "TradeDate" : "200507" 10 | }, 11 | { 12 | "Security": "MSFT", 13 | "LastQty" : 20, 14 | "LastPx" : 183.851234, 15 | "Side" : 1, 16 | "TrdSubType" : 0, 17 | "TradeDate" : "200507" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /finra/openfaas/trddate/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/trddate/requirements.txt -------------------------------------------------------------------------------- /finra/openfaas/volume.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | volume: 7 | lang: python3-flask 8 | handler: ./volume 9 | image: volume:latest 10 | labels: 11 | "com.openfaas.scale.min": "1" 12 | "com.openfaas.scale.max": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /finra/openfaas/volume/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/volume/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/volume/handler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | # from function.util import * 4 | 5 | def timestamp(response, event, startTime, endTime, externalServicesTime): 6 | stampBegin = 1000*time.time() 7 | prior = event['duration'] if 'duration' in event else 0 8 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 9 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 10 | response['duration'] = prior + endTime - startTime 11 | response['workflowEndTime'] = endTime 12 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 13 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 14 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 15 | response['memsetTime'] = priorMemSet 16 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 17 | return response 18 | 19 | def handle(req): 20 | """handle a request to the function 21 | Args: 22 | req (str): request body 23 | """ 24 | event = json.loads(req) 25 | 26 | startTime = 1000*time.time() 27 | 28 | portfolio = event['body']['portfolio'] 29 | portfolios = json.loads(open('/home/app/function/portfolios.json', 'r').read()) 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | qty = str(trade['LastQty']) 36 | # Tag ID: 32, Tag Name: LastQty, Format: max 8 characters, no decimal 37 | if (len(qty)>8) or ('.'in qty): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid':valid, 'portfolio': portfolio}} 42 | endTime = 1000*time.time() 43 | return json.dumps(timestamp(response, event, startTime, endTime, 0)) 44 | -------------------------------------------------------------------------------- /finra/openfaas/volume/portfolios.json: -------------------------------------------------------------------------------- 1 | { 2 | "1234" : [ 3 | { 4 | "Security": "GOOG", 5 | "LastQty" : 10, 6 | "LastPx" : 1363.85123, 7 | "Side" : 1, 8 | "TrdSubType" : 0, 9 | "TradeDate" : "200507" 10 | }, 11 | { 12 | "Security": "MSFT", 13 | "LastQty" : 20, 14 | "LastPx" : 183.851234, 15 | "Side" : 1, 16 | "TrdSubType" : 0, 17 | "TradeDate" : "200507" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /finra/openfaas/volume/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/volume/requirements.txt -------------------------------------------------------------------------------- /finra/openfaas/yfinance.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | yfinance: 7 | lang: python3-flask 8 | handler: ./yfinance 9 | image: yfinance:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /finra/openfaas/yfinance/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/yfinance/__init__.py -------------------------------------------------------------------------------- /finra/openfaas/yfinance/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/openfaas/yfinance/requirements.txt -------------------------------------------------------------------------------- /finra/ray/Workflow.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import threading 3 | 4 | import json 5 | import time 6 | from flask import Flask 7 | 8 | app = Flask(__name__) 9 | 10 | headers = { 11 | "Content-Type": "application/json", 12 | "accept": "application/json", 13 | } 14 | 15 | margin_balance_url = "http://10.244.0.183:8000/fi/marginBalance" 16 | 17 | funcs = ["marketdata", "lastpx", "side", "trddate", "volume"] 18 | 19 | urls = [f"http://10.244.0.183:8000/fi/{func}" for func in funcs] 20 | 21 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 22 | 23 | 24 | def post(url, data, l): 25 | res = requests.post(url, data=data, headers=headers).text 26 | print(url, res) 27 | l.append(res) 28 | 29 | 30 | def workflow(): 31 | start = time.time() 32 | 33 | parallel_res = [] 34 | threads = [] 35 | 36 | for url in urls: 37 | t = threading.Thread(target=post, args=(url, json.dumps(req), parallel_res)) 38 | threads.append(t) 39 | 40 | for t in threads: 41 | t.start() 42 | 43 | for t in threads: 44 | t.join() 45 | 46 | result = [] 47 | 48 | for i in parallel_res: 49 | # print(i) 50 | result.append(json.loads(json.loads(i))) 51 | 52 | print("result::::::::::::::::::") 53 | print(result) 54 | 55 | res = requests.post(margin_balance_url, data=json.dumps(result), headers=headers).text 56 | print("res:::::::::::::::::") 57 | print(res) 58 | # print("This request uses %d so" % int((time.time() - start) * 1000)) 59 | return int((time.time() - start) * 1000) 60 | 61 | 62 | 63 | 64 | @app.route('/hi') 65 | def index(): 66 | lats = [] 67 | for i in range(1): 68 | lat = workflow() 69 | lats.append(lat) 70 | 71 | # print(lats) 72 | # print(sum(lats[1:]) / (len(lats) - 1)) 73 | # print("This request uses %d so" %(sum(lats[1:]) / (len(lats) - 1))) 74 | return { 75 | "msg": "success", 76 | "data": "welcome" 77 | } 78 | 79 | 80 | if __name__ == "__main__": 81 | app.run() -------------------------------------------------------------------------------- /finra/ray/fi/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | class PostItem(BaseModel): 4 | portfolioType: str 5 | portfolio: str 6 | 7 | 8 | class PostStr(BaseModel): 9 | input_str: str 10 | 11 | 12 | class PostBodyMap(BaseModel): 13 | input_name: str 14 | input_part: int 15 | reduce_num: int 16 | 17 | 18 | class PostBodyReduce(BaseModel): 19 | input_name: str 20 | input_num: int 21 | reduce_part: int 22 | -------------------------------------------------------------------------------- /finra/ray/fi/service/market/shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # yfinance - market data downloader 5 | # https://github.com/ranaroussi/yfinance 6 | # 7 | # Copyright 2017-2019 Ran Aroussi 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | _DFS = {} 23 | _PROGRESS_BAR = None 24 | _ERRORS = {} 25 | _ISINS = {} 26 | -------------------------------------------------------------------------------- /finra/ray/fi/service/marketdata.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import time 4 | 5 | from fi.service.market.base import Ticker 6 | 7 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 8 | class MarketdataService(object): 9 | # def __init__(self): 10 | def Marketdata(self, body: Dict): 11 | """handle a request to the function 12 | Args: 13 | req (str): request body 14 | """ 15 | # event = json.loads(body) 16 | print(123) 17 | 18 | 19 | try: 20 | startTime = time.time() 21 | externalServicesTime = [] 22 | portfolioType = body['body']['portfolioType'] 23 | 24 | tickersForPortfolioTypes = {'S&P': ['GOOG', 'AMZN', 'MSFT']} 25 | tickers = tickersForPortfolioTypes[portfolioType] 26 | print("tickers:") 27 | print(tickers) 28 | print(len(tickers)) 29 | prices = {} 30 | for ticker in tickers: 31 | tickerObj = Ticker(ticker) 32 | # Get last closing price 33 | tickTime = time.time() 34 | data = tickerObj.history(period="1") 35 | print("data:") 36 | print(data) 37 | externalServicesTime.append(time.time() - tickTime) 38 | price = data['Close'].unique()[0] 39 | prices[ticker] = price 40 | 41 | # prices = {'GOOG': 1732.38, 'AMZN': 3185.27, 'MSFT': 221.02} 42 | 43 | endTime = time.time() 44 | 45 | response = {'time': {'start': startTime, 'end': endTime, 'externalServicesTime': externalServicesTime}, 46 | 'body': {'marketData': prices}} 47 | print(response) 48 | # return response 49 | 50 | 51 | except Exception as e: 52 | print(e) 53 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 54 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 55 | else: 56 | print("success") 57 | 58 | 59 | return response 60 | 61 | -------------------------------------------------------------------------------- /finra/ray/fi/service/message.py: -------------------------------------------------------------------------------- 1 | def GetPortfolios(): 2 | portfolios = { 3 | "1234": [ 4 | { 5 | "Security": "GOOG", 6 | "LastQty": 10, 7 | "LastPx": 1363.85123, 8 | "Side": 1, 9 | "TrdSubType": 0, 10 | "TradeDate": "200507" 11 | }, 12 | { 13 | "Security": "MSFT", 14 | "LastQty": 20, 15 | "LastPx": 183.851234, 16 | "Side": 1, 17 | "TrdSubType": 0, 18 | "TradeDate": "200507" 19 | } 20 | ] 21 | } 22 | return portfolios 23 | 24 | 25 | def GetMarginjson(): 26 | mj = { 27 | "1234": 4500 28 | } 29 | return mj 30 | -------------------------------------------------------------------------------- /finra/ray/fi/service/side.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | 4 | from fi.service.message import GetPortfolios 5 | from typing import List, Dict 6 | 7 | def timestamp(response, event, startTime, endTime, externalServicesTime): 8 | stampBegin = 1000*time.time() 9 | prior = event['duration'] if 'duration' in event else 0 10 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 11 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 12 | response['duration'] = prior + endTime - startTime 13 | response['workflowEndTime'] = endTime 14 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 15 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 16 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 17 | response['memsetTime'] = priorMemSet 18 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 19 | return response 20 | 21 | 22 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 23 | class SideService(object): 24 | def Side(self, body: Dict): 25 | try: 26 | 27 | startTime = 1000 * time.time() 28 | portfolio = body['body']['portfolio'] 29 | portfolios = GetPortfolios() 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | side = trade['Side'] 36 | # Tag ID: 552, Tag Name: Side, Valid values: 1,2,8 37 | if not (side == 1 or side == 2 or side == 8): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 42 | endTime = 1000 * time.time() 43 | result = timestamp(response, body, startTime, endTime, 0) 44 | print(result) 45 | 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | return result -------------------------------------------------------------------------------- /finra/ray/fi/service/test.py: -------------------------------------------------------------------------------- 1 | from benchmark.ray.finra.fi.service.message import GetPortfolios 2 | 3 | if __name__ == '__main__': 4 | zz = [{"body": {"portfolioType": "S&P", "portfolio": "1234"}},{"body": {"portfolioType": "S&P", "portfolio": "1234"}},{"body": {"portfolioType": "S&P", "portfolio": "1234"}}] 5 | print(type(zz)) 6 | for lab in zz: 7 | print(type(lab)) 8 | 9 | -------------------------------------------------------------------------------- /finra/ray/fi/service/trddate.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | 7 | from fi.service.message import GetPortfolios 8 | from typing import List, Dict 9 | 10 | def timestamp(response, event, startTime, endTime, externalServicesTime): 11 | stampBegin = 1000 * time.time() 12 | prior = event['duration'] if 'duration' in event else 0 13 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 14 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 15 | response['duration'] = prior + endTime - startTime 16 | response['workflowEndTime'] = endTime 17 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 18 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 19 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 20 | response['memsetTime'] = priorMemSet 21 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 22 | return response 23 | 24 | 25 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 26 | class TrddateService(object): 27 | 28 | def Trddate(self, body: Dict): 29 | try: 30 | 31 | startTime = 1000 * time.time() 32 | 33 | portfolio = body['body']['portfolio'] 34 | portfolios = GetPortfolios() 35 | data = portfolios[portfolio] 36 | 37 | valid = True 38 | 39 | for trade in data: 40 | trddate = trade['TradeDate'] 41 | # Tag ID: 75, Tag Name: TradeDate, Format: YYMMDD 42 | if len(trddate) == 6: 43 | try: 44 | datetime.datetime(int(trddate[0:2]), int(trddate[2:4]), int(trddate[4:6])) 45 | except ValueError: 46 | valid = False 47 | break 48 | else: 49 | valid = False 50 | break 51 | 52 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 53 | endTime = 1000 * time.time() 54 | 55 | result = timestamp(response, body, startTime, endTime, 0) 56 | print(result) 57 | 58 | except Exception as e: 59 | print(e) 60 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 61 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 62 | 63 | else: 64 | print("success") 65 | 66 | return result 67 | -------------------------------------------------------------------------------- /finra/ray/fi/service/volume.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | from fi.service.message import GetPortfolios 4 | from typing import List, Dict 5 | 6 | def timestamp(response, event, startTime, endTime, externalServicesTime): 7 | stampBegin = 1000 * time.time() 8 | prior = event['duration'] if 'duration' in event else 0 9 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 10 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 11 | response['duration'] = prior + endTime - startTime 12 | response['workflowEndTime'] = endTime 13 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 14 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 15 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 16 | response['memsetTime'] = priorMemSet 17 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 18 | return response 19 | 20 | 21 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 22 | class VolumeService(object): 23 | 24 | def Volume(self, body: Dict): 25 | try: 26 | startTime = 1000 * time.time() 27 | 28 | portfolio = body['body']['portfolio'] 29 | portfolios = GetPortfolios() 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | qty = str(trade['LastQty']) 36 | # Tag ID: 32, Tag Name: LastQty, Format: max 8 characters, no decimal 37 | if (len(qty) > 8) or ('.' in qty): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 42 | endTime = 1000 * time.time() 43 | result = timestamp(response, body, startTime, endTime, 0) 44 | print(result) 45 | 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | return result 54 | -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/marketdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/ray/fi/service_bak/marketdata.py -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/message.py: -------------------------------------------------------------------------------- 1 | def GetPortfolios(): 2 | portfolios = { 3 | "1234": [ 4 | { 5 | "Security": "GOOG", 6 | "LastQty": 10, 7 | "LastPx": 1363.85123, 8 | "Side": 1, 9 | "TrdSubType": 0, 10 | "TradeDate": "200507" 11 | }, 12 | { 13 | "Security": "MSFT", 14 | "LastQty": 20, 15 | "LastPx": 183.851234, 16 | "Side": 1, 17 | "TrdSubType": 0, 18 | "TradeDate": "200507" 19 | } 20 | ] 21 | } 22 | return portfolios 23 | -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/side.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | 4 | from fi.model.post import PostItem 5 | from fastapi import Body 6 | from fi.service.message import GetPortfolios 7 | 8 | 9 | def timestamp(response, event, startTime, endTime, externalServicesTime): 10 | stampBegin = 1000*time.time() 11 | prior = event['duration'] if 'duration' in event else 0 12 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 13 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 14 | response['duration'] = prior + endTime - startTime 15 | response['workflowEndTime'] = endTime 16 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 17 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 18 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 19 | response['memsetTime'] = priorMemSet 20 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 21 | return response 22 | 23 | 24 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 25 | class SideService(object): 26 | def Side(self, body: PostItem = Body(embed=True)): 27 | startTime = 1000 * time.time() 28 | portfolio = body.portfolio 29 | portfolios = GetPortfolios() 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | side = trade['Side'] 36 | # Tag ID: 552, Tag Name: Side, Valid values: 1,2,8 37 | if not (side == 1 or side == 2 or side == 8): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 42 | endTime = 1000 * time.time() 43 | print(timestamp(response, body, startTime, endTime, 0)) 44 | return timestamp(response, body, startTime, endTime, 0) -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/test.py: -------------------------------------------------------------------------------- 1 | from benchmark.ray.finra.fi.service.message import GetPortfolios 2 | 3 | if __name__ == '__main__': 4 | aaa = GetPortfolios() 5 | print(aaa["1234"]) -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/trddate.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | from fi.model.post import PostItem 7 | from fastapi import Body 8 | from fi.service.message import GetPortfolios 9 | 10 | 11 | def timestamp(response, event, startTime, endTime, externalServicesTime): 12 | stampBegin = 1000 * time.time() 13 | prior = event['duration'] if 'duration' in event else 0 14 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 15 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 16 | response['duration'] = prior + endTime - startTime 17 | response['workflowEndTime'] = endTime 18 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 19 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 20 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 21 | response['memsetTime'] = priorMemSet 22 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 23 | return response 24 | 25 | 26 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 27 | class TrddateService(object): 28 | 29 | def Trddate(self, body: PostItem = Body(embed=True)): 30 | startTime = 1000 * time.time() 31 | 32 | 33 | 34 | portfolio = body.portfolio 35 | 36 | portfolios = GetPortfolios() 37 | print(type(portfolios)) 38 | data = portfolios[portfolio] 39 | print(data) 40 | 41 | valid = True 42 | 43 | for trade in data: 44 | trddate = trade['TradeDate'] 45 | # Tag ID: 75, Tag Name: TradeDate, Format: YYMMDD 46 | if len(trddate) == 6: 47 | try: 48 | datetime.datetime(int(trddate[0:2]), int(trddate[2:4]), int(trddate[4:6])) 49 | except ValueError: 50 | valid = False 51 | break 52 | else: 53 | valid = False 54 | break 55 | 56 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 57 | endTime = 1000 * time.time() 58 | return timestamp(response, body, startTime, endTime, 0) 59 | -------------------------------------------------------------------------------- /finra/ray/fi/service_bak/volume.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | from fi.model.post import PostItem 7 | from fastapi import Body 8 | from fi.service.message import GetPortfolios 9 | 10 | 11 | def timestamp(response, event, startTime, endTime, externalServicesTime): 12 | stampBegin = 1000 * time.time() 13 | prior = event['duration'] if 'duration' in event else 0 14 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 15 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 16 | response['duration'] = prior + endTime - startTime 17 | response['workflowEndTime'] = endTime 18 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 19 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 20 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 21 | response['memsetTime'] = priorMemSet 22 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 23 | return response 24 | 25 | 26 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 27 | class VolumeService(object): 28 | 29 | def Volume(self, body: PostItem = Body(embed=True)): 30 | startTime = 1000 * time.time() 31 | 32 | 33 | 34 | portfolio = body.portfolio 35 | 36 | portfolios = GetPortfolios() 37 | print(type(portfolios)) 38 | data = portfolios[portfolio] 39 | print(data) 40 | 41 | valid = True 42 | 43 | for trade in data: 44 | qty = str(trade['LastQty']) 45 | # Tag ID: 32, Tag Name: LastQty, Format: max 8 characters, no decimal 46 | if (len(qty) > 8) or ('.' in qty): 47 | valid = False 48 | break 49 | 50 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 51 | endTime = 1000 * time.time() 52 | return timestamp(response, body, startTime, endTime, 0) 53 | -------------------------------------------------------------------------------- /finra/ray/req.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | headers = { 5 | "Content-Type": "application/json", 6 | "accept": "application/json", 7 | } 8 | 9 | 10 | #req = '{"body":{ "portfolioType":"S&P", "portfolio":"1234"}}' 11 | 12 | 13 | # req = { "portfolioType":"S&P", "portfolio":"1234"} 14 | 15 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 16 | resp3 = requests.post("http://10.244.0.158:8000/fi/marketdata", data=json.dumps(req), headers=headers) 17 | 18 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 19 | resp3 = requests.post("http://10.244.0.158:8000/fi/lastpx", data=json.dumps(req), headers=headers) 20 | 21 | print(resp3.text) 22 | 23 | resp3 = requests.post("http://10.244.0.158:8000/fi/side", data=json.dumps(req), headers=headers) 24 | 25 | print(resp3.text) 26 | 27 | resp3 = requests.post("http://10.244.0.158:8000/fi/trddate", data=json.dumps(req), headers=headers) 28 | 29 | print(resp3.text) 30 | resp3 = requests.post("http://10.244.0.158:8000/fi/volume", data=json.dumps(req), headers=headers) 31 | 32 | print(resp3.text) 33 | 34 | # req = {"body":"MSFT"} 35 | # 36 | # resp3 = requests.post("http://10.244.0.158:8000/fi/yfinance", data=json.dumps(req), headers=headers) 37 | 38 | print(resp3.text) 39 | 40 | -------------------------------------------------------------------------------- /finra/ray_stateful/Workflow.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import threading 3 | 4 | import json 5 | import time 6 | from flask import Flask 7 | 8 | app = Flask(__name__) 9 | 10 | headers = { 11 | "Content-Type": "application/json", 12 | "accept": "application/json", 13 | } 14 | 15 | margin_balance_url = "http://10.244.0.183:8000/fi/marginBalance" 16 | 17 | funcs = ["marketdata", "lastpx", "side", "trddate", "volume"] 18 | 19 | urls = [f"http://10.244.0.183:8000/fi/{func}" for func in funcs] 20 | 21 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 22 | 23 | 24 | def post(url, data, l): 25 | res = requests.post(url, data=data, headers=headers).text 26 | print(url, res) 27 | l.append(res) 28 | 29 | 30 | def workflow(): 31 | start = time.time() 32 | 33 | parallel_res = [] 34 | threads = [] 35 | 36 | for url in urls: 37 | t = threading.Thread(target=post, args=(url, json.dumps(req), parallel_res)) 38 | threads.append(t) 39 | 40 | for t in threads: 41 | t.start() 42 | 43 | for t in threads: 44 | t.join() 45 | 46 | result = [] 47 | 48 | for i in parallel_res: 49 | # print(i) 50 | result.append(json.loads(json.loads(i))) 51 | 52 | print("result::::::::::::::::::") 53 | print(result) 54 | 55 | res = requests.post(margin_balance_url, data=json.dumps(result), headers=headers).text 56 | print("res:::::::::::::::::") 57 | print(res) 58 | # print("This request uses %d so" % int((time.time() - start) * 1000)) 59 | return int((time.time() - start) * 1000) 60 | 61 | 62 | 63 | 64 | @app.route('/hi') 65 | def index(): 66 | lats = [] 67 | for i in range(1): 68 | lat = workflow() 69 | lats.append(lat) 70 | 71 | # print(lats) 72 | # print(sum(lats[1:]) / (len(lats) - 1)) 73 | # print("This request uses %d so" %(sum(lats[1:]) / (len(lats) - 1))) 74 | return { 75 | "msg": "success", 76 | "data": "welcome" 77 | } 78 | 79 | 80 | if __name__ == "__main__": 81 | app.run() -------------------------------------------------------------------------------- /finra/ray_stateful/fi/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | class PostItem(BaseModel): 4 | portfolioType: str 5 | portfolio: str 6 | 7 | 8 | class PostStr(BaseModel): 9 | input_str: str 10 | 11 | 12 | class PostBodyMap(BaseModel): 13 | input_name: str 14 | input_part: int 15 | reduce_num: int 16 | 17 | 18 | class PostBodyReduce(BaseModel): 19 | input_name: str 20 | input_num: int 21 | reduce_part: int 22 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/market/1: -------------------------------------------------------------------------------- 1 | global financedata 2 | if ticker in financedata: 3 | tickerObj = financedata[ticker] 4 | print("存在") 5 | else: 6 | tickerObj = Ticker(ticker) 7 | print("不存在") 8 | financedata[ticker] = tickerObj 9 | 10 | global a 11 | a += 1 12 | print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", a) -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/market/shared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # yfinance - market data downloader 5 | # https://github.com/ranaroussi/yfinance 6 | # 7 | # Copyright 2017-2019 Ran Aroussi 8 | # 9 | # Licensed under the Apache License, Version 2.0 (the "License"); 10 | # you may not use this file except in compliance with the License. 11 | # You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | # 21 | 22 | _DFS = {} 23 | _PROGRESS_BAR = None 24 | _ERRORS = {} 25 | _ISINS = {} 26 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/message.py: -------------------------------------------------------------------------------- 1 | def GetPortfolios(): 2 | portfolios = { 3 | "1234": [ 4 | { 5 | "Security": "GOOG", 6 | "LastQty": 10, 7 | "LastPx": 1363.85123, 8 | "Side": 1, 9 | "TrdSubType": 0, 10 | "TradeDate": "200507" 11 | }, 12 | { 13 | "Security": "MSFT", 14 | "LastQty": 20, 15 | "LastPx": 183.851234, 16 | "Side": 1, 17 | "TrdSubType": 0, 18 | "TradeDate": "200507" 19 | } 20 | ] 21 | } 22 | return portfolios 23 | 24 | 25 | def GetMarginjson(): 26 | mj = { 27 | "1234": 4500 28 | } 29 | return mj 30 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/side.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | 4 | from fi.service.message import GetPortfolios 5 | from typing import List, Dict 6 | 7 | def timestamp(response, event, startTime, endTime, externalServicesTime): 8 | stampBegin = 1000*time.time() 9 | prior = event['duration'] if 'duration' in event else 0 10 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 11 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 12 | response['duration'] = prior + endTime - startTime 13 | response['workflowEndTime'] = endTime 14 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 15 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 16 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 17 | response['memsetTime'] = priorMemSet 18 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 19 | return response 20 | 21 | 22 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 23 | class SideService(object): 24 | def Side(self, body: Dict): 25 | try: 26 | 27 | startTime = 1000 * time.time() 28 | portfolio = body['body']['portfolio'] 29 | portfolios = GetPortfolios() 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | side = trade['Side'] 36 | # Tag ID: 552, Tag Name: Side, Valid values: 1,2,8 37 | if not (side == 1 or side == 2 or side == 8): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 42 | endTime = 1000 * time.time() 43 | result = timestamp(response, body, startTime, endTime, 0) 44 | print(result) 45 | 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | return result -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/test.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | if __name__ == '__main__': 4 | zz = [{"body": {"portfolioType": "S&P", "portfolio": "1234"}},{"body": {"portfolioType": "S&P", "portfolio": "1234"}},{"body": {"portfolioType": "S&P", "portfolio": "1234"}}] 5 | aa = {"31":31} 6 | if isinstance(aa,dict): 7 | print(1) 8 | 9 | 10 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/trddate.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | 7 | from fi.service.message import GetPortfolios 8 | from typing import List, Dict 9 | 10 | def timestamp(response, event, startTime, endTime, externalServicesTime): 11 | stampBegin = 1000 * time.time() 12 | prior = event['duration'] if 'duration' in event else 0 13 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 14 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 15 | response['duration'] = prior + endTime - startTime 16 | response['workflowEndTime'] = endTime 17 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 18 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 19 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 20 | response['memsetTime'] = priorMemSet 21 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 22 | return response 23 | 24 | 25 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 26 | class TrddateService(object): 27 | 28 | def Trddate(self, body: Dict): 29 | try: 30 | 31 | startTime = 1000 * time.time() 32 | 33 | portfolio = body['body']['portfolio'] 34 | portfolios = GetPortfolios() 35 | data = portfolios[portfolio] 36 | 37 | valid = True 38 | 39 | for trade in data: 40 | trddate = trade['TradeDate'] 41 | # Tag ID: 75, Tag Name: TradeDate, Format: YYMMDD 42 | if len(trddate) == 6: 43 | try: 44 | datetime.datetime(int(trddate[0:2]), int(trddate[2:4]), int(trddate[4:6])) 45 | except ValueError: 46 | valid = False 47 | break 48 | else: 49 | valid = False 50 | break 51 | 52 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 53 | endTime = 1000 * time.time() 54 | 55 | result = timestamp(response, body, startTime, endTime, 0) 56 | print(result) 57 | 58 | except Exception as e: 59 | print(e) 60 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 61 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 62 | 63 | else: 64 | print("success") 65 | 66 | return result 67 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service/volume.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | from fi.service.message import GetPortfolios 4 | from typing import List, Dict 5 | 6 | def timestamp(response, event, startTime, endTime, externalServicesTime): 7 | stampBegin = 1000 * time.time() 8 | prior = event['duration'] if 'duration' in event else 0 9 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 10 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 11 | response['duration'] = prior + endTime - startTime 12 | response['workflowEndTime'] = endTime 13 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 14 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 15 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 16 | response['memsetTime'] = priorMemSet 17 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 18 | return response 19 | 20 | 21 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 22 | class VolumeService(object): 23 | 24 | def Volume(self, body: Dict): 25 | try: 26 | startTime = 1000 * time.time() 27 | 28 | portfolio = body['body']['portfolio'] 29 | portfolios = GetPortfolios() 30 | data = portfolios[portfolio] 31 | 32 | valid = True 33 | 34 | for trade in data: 35 | qty = str(trade['LastQty']) 36 | # Tag ID: 32, Tag Name: LastQty, Format: max 8 characters, no decimal 37 | if (len(qty) > 8) or ('.' in qty): 38 | valid = False 39 | break 40 | 41 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 42 | endTime = 1000 * time.time() 43 | result = timestamp(response, body, startTime, endTime, 0) 44 | print(result) 45 | 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | return result 54 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/marketdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/finra/ray_stateful/fi/service_bak/marketdata.py -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/message.py: -------------------------------------------------------------------------------- 1 | def GetPortfolios(): 2 | portfolios = { 3 | "1234": [ 4 | { 5 | "Security": "GOOG", 6 | "LastQty": 10, 7 | "LastPx": 1363.85123, 8 | "Side": 1, 9 | "TrdSubType": 0, 10 | "TradeDate": "200507" 11 | }, 12 | { 13 | "Security": "MSFT", 14 | "LastQty": 20, 15 | "LastPx": 183.851234, 16 | "Side": 1, 17 | "TrdSubType": 0, 18 | "TradeDate": "200507" 19 | } 20 | ] 21 | } 22 | return portfolios 23 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/side.py: -------------------------------------------------------------------------------- 1 | import time 2 | from ray import serve 3 | 4 | from fi.model.post import PostItem 5 | from fastapi import Body 6 | from fi.service.message import GetPortfolios 7 | import ray 8 | from typing import List, Dict 9 | 10 | def timestamp(response, event, startTime, endTime, externalServicesTime): 11 | stampBegin = 1000*time.time() 12 | prior = event['duration'] if 'duration' in event else 0 13 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 14 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 15 | response['duration'] = prior + endTime - startTime 16 | response['workflowEndTime'] = endTime 17 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 18 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 19 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 20 | response['memsetTime'] = priorMemSet 21 | response['timeStampCost'] = priorCost - (stampBegin-1000*time.time()) 22 | return response 23 | 24 | 25 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 26 | class SideService(object): 27 | def Side(self, body: Dict): 28 | startTime = 1000 * time.time() 29 | portfolio = body["portfolio"] 30 | portfolios = GetPortfolios() 31 | data = portfolios[portfolio] 32 | 33 | valid = True 34 | 35 | for trade in data: 36 | side = trade['Side'] 37 | # Tag ID: 552, Tag Name: Side, Valid values: 1,2,8 38 | if not (side == 1 or side == 2 or side == 8): 39 | valid = False 40 | break 41 | 42 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 43 | endTime = 1000 * time.time() 44 | print(timestamp(response, body, startTime, endTime, 0)) 45 | result = timestamp(response, body, startTime, endTime, 0) 46 | return ray.put(result) -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/test.py: -------------------------------------------------------------------------------- 1 | from benchmark.ray.finra.fi.service.message import GetPortfolios 2 | 3 | if __name__ == '__main__': 4 | aaa = GetPortfolios() 5 | print(aaa["1234"]) -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/trddate.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | from fi.model.post import PostItem 7 | from fastapi import Body 8 | from fi.service.message import GetPortfolios 9 | import ray 10 | from typing import List, Dict 11 | 12 | def timestamp(response, event, startTime, endTime, externalServicesTime): 13 | stampBegin = 1000 * time.time() 14 | prior = event['duration'] if 'duration' in event else 0 15 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 16 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 17 | response['duration'] = prior + endTime - startTime 18 | response['workflowEndTime'] = endTime 19 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 20 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 21 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 22 | response['memsetTime'] = priorMemSet 23 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 24 | return response 25 | 26 | 27 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 28 | class TrddateService(object): 29 | 30 | def Trddate(self, body: PostItem = Dict): 31 | startTime = 1000 * time.time() 32 | 33 | 34 | 35 | portfolio = body["portfolio"] 36 | 37 | portfolios = GetPortfolios() 38 | print(type(portfolios)) 39 | data = portfolios[portfolio] 40 | print(data) 41 | 42 | valid = True 43 | 44 | for trade in data: 45 | trddate = trade['TradeDate'] 46 | # Tag ID: 75, Tag Name: TradeDate, Format: YYMMDD 47 | if len(trddate) == 6: 48 | try: 49 | datetime.datetime(int(trddate[0:2]), int(trddate[2:4]), int(trddate[4:6])) 50 | except ValueError: 51 | valid = False 52 | break 53 | else: 54 | valid = False 55 | break 56 | 57 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 58 | endTime = 1000 * time.time() 59 | result = timestamp(response, body, startTime, endTime, 0) 60 | return ray.put(result) 61 | -------------------------------------------------------------------------------- /finra/ray_stateful/fi/service_bak/volume.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import datetime 4 | 5 | from ray import serve 6 | from fi.model.post import PostItem 7 | from fastapi import Body 8 | from fi.service.message import GetPortfolios 9 | import ray 10 | from typing import List, Dict 11 | 12 | def timestamp(response, event, startTime, endTime, externalServicesTime): 13 | stampBegin = 1000 * time.time() 14 | prior = event['duration'] if 'duration' in event else 0 15 | priorMemSet = event['memsetTime'] if 'memsetTime' in event else 0 16 | priorServiceTime = event['externalServicesTime'] if 'externalServicesTime' in event else 0 17 | response['duration'] = prior + endTime - startTime 18 | response['workflowEndTime'] = endTime 19 | response['workflowStartTime'] = event['workflowStartTime'] if 'workflowStartTime' in event else startTime 20 | priorCost = event['timeStampCost'] if 'timeStampCost' in event else 0 21 | response['externalServicesTime'] = priorServiceTime + externalServicesTime 22 | response['memsetTime'] = priorMemSet 23 | response['timeStampCost'] = priorCost - (stampBegin - 1000 * time.time()) 24 | return response 25 | 26 | 27 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 28 | class VolumeService(object): 29 | 30 | def Volume(self, body: Dict): 31 | startTime = 1000 * time.time() 32 | 33 | 34 | 35 | portfolio = body["portfolio"] 36 | 37 | portfolios = GetPortfolios() 38 | print(type(portfolios)) 39 | data = portfolios[portfolio] 40 | print(data) 41 | 42 | valid = True 43 | 44 | for trade in data: 45 | qty = str(trade['LastQty']) 46 | # Tag ID: 32, Tag Name: LastQty, Format: max 8 characters, no decimal 47 | if (len(qty) > 8) or ('.' in qty): 48 | valid = False 49 | break 50 | 51 | response = {'statusCode': 200, 'body': {'valid': valid, 'portfolio': portfolio}} 52 | endTime = 1000 * time.time() 53 | result = timestamp(response, body, startTime, endTime, 0) 54 | return ray.put(result) 55 | -------------------------------------------------------------------------------- /finra/ray_stateful/req.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | headers = { 5 | "Content-Type": "application/json", 6 | "accept": "application/json", 7 | } 8 | 9 | 10 | #req = '{"body":{ "portfolioType":"S&P", "portfolio":"1234"}}' 11 | 12 | 13 | # req = { "portfolioType":"S&P", "portfolio":"1234"} 14 | 15 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 16 | resp3 = requests.post("http://10.244.0.158:8000/fi/marketdata", data=json.dumps(req), headers=headers) 17 | 18 | req = {"body":{ "portfolioType":"S&P", "portfolio":"1234"}} 19 | resp3 = requests.post("http://10.244.0.158:8000/fi/lastpx", data=json.dumps(req), headers=headers) 20 | 21 | print(resp3.text) 22 | 23 | resp3 = requests.post("http://10.244.0.158:8000/fi/side", data=json.dumps(req), headers=headers) 24 | 25 | print(resp3.text) 26 | 27 | resp3 = requests.post("http://10.244.0.158:8000/fi/trddate", data=json.dumps(req), headers=headers) 28 | 29 | print(resp3.text) 30 | resp3 = requests.post("http://10.244.0.158:8000/fi/volume", data=json.dumps(req), headers=headers) 31 | 32 | print(resp3.text) 33 | 34 | # req = {"body":"MSFT"} 35 | # 36 | # resp3 = requests.post("http://10.244.0.158:8000/fi/yfinance", data=json.dumps(req), headers=headers) 37 | 38 | print(resp3.text) 39 | 40 | -------------------------------------------------------------------------------- /jhdaslkd: -------------------------------------------------------------------------------- 1 | dev 2 | * dev_pyz 3 | main 4 | remotes/origin/HEAD -> origin/main 5 | remotes/origin/dev 6 | remotes/origin/dev_pyz 7 | remotes/origin/main 8 | -------------------------------------------------------------------------------- /mapreduce/openfaas/README.md: -------------------------------------------------------------------------------- 1 | map reduce 2 | 3 | 环境配置 4 | cd yaml 5 | # 部署Redis 6 | kubectl apply -f redis.yaml 7 | # 部署MinIO 8 | kubectl apply -f minio.yaml 9 | 注意事项： 10 | namespace：默认为default，建议改为自己的ns以免冲突 11 | nodeName：请改成自己集群的节点，或者删除不绑定 12 | minio的挂载路径：默认为/mnt/minio，请修改或者自行创建相关目录 13 | minio的用户名和密码：有需要请修改MINIO_ROOT_USER和MINIO_ROOT_PASSWORD，并相应修改下文3中的函数中的配置 14 | 15 | 数据生成 16 | cd gen_data 17 | # 编译 18 | cd . 19 | # 生成数据，参数依次代表模型名，文件数量，文件行数，每行词数，输出目录，可根据需要更改 20 | ./gen_text_data.sh lda_wiki1w 5 8000 2000 wc-100M-5/ 21 | # 上传数据至存储系统，注意修改对应参数。请自行调用make_bucket API创建数据桶。 22 | python3 upload.py 23 | 24 | 实验运行 25 | cd functions 26 | # 编译镜像 27 | faashwh-cli build -f wc-mapper.yml 28 | faashwh-cli build -f wc-reducer.yml 29 | # 部署 30 | faashwh-cli deploy -f wc-mapper.yml 31 | faashwh-cli deploy -f wc-reducer.yml 32 | # 运行 33 | python3 client.py 34 | # 清理中间数据 35 | python3 clean.py 36 | 注意事项： 37 | 数据存储：目前Redis和Minio的配置使用默认值，有更改请更新代码并重新编译部署。当前使用MinIO作为输入和输出的存储系统，Redis作为中间数据的存储。 38 | 函数配置：目前锁定5个实例和部署位置。节点限制可自行更改或直接删除。因没有实现输入文件自动分区，所以锁定5个函数实例，暂不支持自定义实例数量。如需修改，需要相应以下参数 39 | 1：gen_text_data中生成文件数量的参数 40 | 2：upload.py脚本中的分区数量 41 | 3：yml文件中实例的数量 42 | 4：client脚本中mapper_num和reducer_num变量 43 | 客户端调用：目前使用默认的输入目录。有需要请自行更改。 44 | 中间数据清理：请自行更新为本地Redis服务的IP。 -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/clean.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | client = redis.Redis(host="192.168.13.175", port=6379) 4 | 5 | client.flushall() 6 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import subprocess 5 | import threading 6 | 7 | mapper_name = "wc-mapper" 8 | reducer_name = "wc-reducer" 9 | mapper_num = 5 10 | reducer_num = 5 11 | 12 | def get_IPs(func_name): 13 | print("Get %s IPs" % func_name) 14 | cmd = "kubectl get pods -n openfaas-fn-hwh -o wide | grep %s- | awk '{print $6}'" % (func_name) 15 | IPs = subprocess.check_output(cmd, shell=True).decode() 16 | IPs = ["http://%s:5000" % IP for IP in IPs.strip().split("\n")] 17 | print(func_name,": ",[i for i in IPs]) 18 | return IPs 19 | 20 | def post(url, data, l): 21 | res = requests.post(url, data).text 22 | l.append(res) 23 | 24 | def workflow(mapper_IPs, reducer_IPs): 25 | start = time.time() 26 | 27 | mapper_res = [] 28 | threads = [] 29 | for i in range(mapper_num): 30 | req = json.dumps({"input_name": "data-500m", "input_part": i, "reduce_num": reducer_num}) 31 | t = threading.Thread(target=post, args=(mapper_IPs[i], req, mapper_res)) 32 | threads.append(t) 33 | 34 | for t in threads: 35 | t.start() 36 | 37 | for t in threads: 38 | t.join() 39 | 40 | mapper_end = time.time() 41 | 42 | reducer_res = [] 43 | threads = [] 44 | for i in range(reducer_num): 45 | req = json.dumps({"input_name": "data-500m", "input_num": mapper_num, "reduce_part": i}) 46 | t = threading.Thread(target=post, args=(reducer_IPs[i], req, reducer_res)) 47 | threads.append(t) 48 | 49 | for t in threads: 50 | t.start() 51 | 52 | for t in threads: 53 | t.join() 54 | 55 | reducer_end = time.time() 56 | 57 | return mapper_res, reducer_res 58 | 59 | def get_res_info(res): 60 | res_j = [json.loads(r) for r in res] 61 | read_times = [] 62 | com_times = [] 63 | store_times = [] 64 | for r in res_j: 65 | read_times.append((r["read_end"] - r["read_start"])*1000) 66 | com_times.append((r["com_end"] - r["read_end"])*1000) 67 | store_times.append((r["store_end"] - r["com_end"])*1000) 68 | 69 | print("Read Time: ", read_times, sum(read_times)/len(read_times)) 70 | print("Compute Time: ", com_times, sum(com_times)/len(com_times)) 71 | print("Store Time: ", store_times, sum(store_times)/len(store_times)) 72 | 73 | if __name__=="__main__": 74 | mapper_IPs = get_IPs(mapper_name) 75 | reducer_IPs = get_IPs(reducer_name) 76 | 77 | mapper_res, reducer_res = workflow(mapper_IPs, reducer_IPs) 78 | 79 | print("===Map Info===",mapper_res) 80 | # get_res_info(mapper_res) 81 | print("===Reduce Info===",reducer_res) 82 | # get_res_info(reducer_res) 83 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/openfaas/of-watchdog:0.7.7 as watchdog 2 | FROM python:3.7-slim-buster 3 | 4 | COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog 5 | RUN chmod +x /usr/bin/fwatchdog 6 | 7 | ARG ADDITIONAL_PACKAGE 8 | # Alternatively use ADD https:// (which will not be cached by Docker builder) 9 | 10 | RUN apt-get -qy update && apt-get -qy install gcc make ${ADDITIONAL_PACKAGE} 11 | 12 | # Add non root user 13 | RUN addgroup --system app && adduser app --system --ingroup app 14 | RUN chown app /home/app 15 | 16 | USER app 17 | 18 | ENV PATH=$PATH:/home/app/.local/bin 19 | 20 | WORKDIR /home/app/ 21 | 22 | COPY index.py . 23 | COPY requirements.txt . 24 | 25 | USER root 26 | RUN pip install -r requirements.txt 27 | 28 | #build function directory and install user specified componenets. 29 | USER app 30 | 31 | RUN mkdir -p function 32 | RUN touch ./function/__init__.py 33 | WORKDIR /home/app/function/ 34 | COPY function/requirements.txt . 35 | RUN pip install --user -r requirements.txt 36 | 37 | WORKDIR /home/app/ 38 | 39 | #install function code 40 | USER root 41 | 42 | COPY function function 43 | RUN chown -R app:app ./ 44 | 45 | #configure WSGI server and healthcheck 46 | USER app 47 | 48 | ENV fprocess="python index.py" 49 | 50 | ENV cgi_headers="true" 51 | ENV mode="http" 52 | ENV upstream_url="http://127.0.0.1:5000" 53 | 54 | HEALTHCHECK --interval=5s CMD [ -e /tmp/.lock ] || exit 1 55 | 56 | CMD ["fwatchdog"] 57 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/template/python3-flask-debian/function/__init__.py -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/function/handler.py: -------------------------------------------------------------------------------- 1 | def handle(req): 2 | """handle a request to the function 3 | Args: 4 | req (str): request body 5 | """ 6 | 7 | return req 8 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/function/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/template/python3-flask-debian/function/requirements.txt -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/index.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Alex Ellis 2017. All rights reserved. 2 | # Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | from flask import Flask, request 5 | from function import handler 6 | from waitress import serve 7 | import os 8 | 9 | app = Flask(__name__) 10 | 11 | # distutils.util.strtobool() can throw an exception 12 | def is_true(val): 13 | return len(val) > 0 and val.lower() == "true" or val == "1" 14 | 15 | @app.before_request 16 | def fix_transfer_encoding(): 17 | """ 18 | Sets the "wsgi.input_terminated" environment flag, thus enabling 19 | Werkzeug to pass chunked requests as streams. The gunicorn server 20 | should set this, but it's not yet been implemented. 21 | """ 22 | 23 | transfer_encoding = request.headers.get("Transfer-Encoding", None) 24 | if transfer_encoding == u"chunked": 25 | request.environ["wsgi.input_terminated"] = True 26 | 27 | @app.route("/", defaults={"path": ""}, methods=["POST", "GET"]) 28 | @app.route("/", methods=["POST", "GET"]) 29 | def main_route(path): 30 | raw_body = os.getenv("RAW_BODY", "false") 31 | 32 | as_text = True 33 | 34 | if is_true(raw_body): 35 | as_text = False 36 | 37 | ret = handler.handle(request.get_data(as_text=as_text)) 38 | return ret 39 | 40 | if __name__ == '__main__': 41 | serve(app, host='0.0.0.0', port=5000) 42 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | waitress 3 | 4 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask-debian/template.yml: -------------------------------------------------------------------------------- 1 | language: python3-flask-debian 2 | fprocess: python index.py 3 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/.Dockerfile.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/template/python3-flask/.Dockerfile.swp -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/openfaas/of-watchdog:0.7.7 as watchdog 2 | FROM python:3.7-alpine 3 | 4 | RUN echo -e http://mirrors.ustc.edu.cn/alpine/v3.7/main/ > /etc/apk/repositories 5 | 6 | COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog 7 | RUN chmod +x /usr/bin/fwatchdog 8 | 9 | ARG ADDITIONAL_PACKAGE 10 | # Alternatively use ADD https:// (which will not be cached by Docker builder) 11 | 12 | RUN apk --no-cache add openssl-dev ${ADDITIONAL_PACKAGE} 13 | 14 | # Add non root user 15 | RUN addgroup -S app && adduser app -S -G app 16 | RUN chown app /home/app 17 | 18 | USER app 19 | 20 | ENV PATH=$PATH:/home/app/.local/bin 21 | 22 | WORKDIR /home/app/ 23 | 24 | COPY index.py . 25 | COPY requirements.txt . 26 | 27 | USER root 28 | RUN pip install -r requirements.txt 29 | 30 | #build function directory and install user specified componenets. 31 | USER app 32 | 33 | RUN mkdir -p function 34 | RUN touch ./function/__init__.py 35 | WORKDIR /home/app/function/ 36 | COPY function/requirements.txt . 37 | RUN pip install --user -r requirements.txt 38 | 39 | WORKDIR /home/app/ 40 | 41 | #install function code 42 | USER root 43 | 44 | COPY function function 45 | RUN chown -R app:app ./ 46 | 47 | #configure WSGI server and healthcheck 48 | USER app 49 | 50 | ENV fprocess="python index.py" 51 | 52 | ENV cgi_headers="true" 53 | ENV mode="http" 54 | ENV upstream_url="http://127.0.0.1:5000" 55 | 56 | HEALTHCHECK --interval=5s CMD [ -e /tmp/.lock ] || exit 1 57 | 58 | CMD ["fwatchdog"] 59 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/function/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/template/python3-flask/function/__init__.py -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/function/handler.py: -------------------------------------------------------------------------------- 1 | def handle(req): 2 | """handle a request to the function 3 | Args: 4 | req (str): request body 5 | """ 6 | 7 | return req 8 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/function/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/template/python3-flask/function/requirements.txt -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/index.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Alex Ellis 2017. All rights reserved. 2 | # Licensed under the MIT license. See LICENSE file in the project root for full license information. 3 | 4 | from flask import Flask, request 5 | from function import handler 6 | from waitress import serve 7 | import os 8 | #import traceback 9 | 10 | app = Flask(__name__) 11 | 12 | # distutils.util.strtobool() can throw an exception 13 | def is_true(val): 14 | return len(val) > 0 and val.lower() == "true" or val == "1" 15 | 16 | @app.before_request 17 | def fix_transfer_encoding(): 18 | """ 19 | Sets the "wsgi.input_terminated" environment flag, thus enabling 20 | Werkzeug to pass chunked requests as streams. The gunicorn server 21 | should set this, but it's not yet been implemented. 22 | """ 23 | 24 | transfer_encoding = request.headers.get("Transfer-Encoding", None) 25 | if transfer_encoding == u"chunked": 26 | request.environ["wsgi.input_terminated"] = True 27 | 28 | @app.route("/", defaults={"path": ""}, methods=["POST", "GET"]) 29 | @app.route("/", methods=["POST", "GET"]) 30 | def main_route(path): 31 | raw_body = os.getenv("RAW_BODY", "false") 32 | 33 | as_text = True 34 | 35 | if is_true(raw_body): 36 | as_text = False 37 | 38 | #try: 39 | ret = handler.handle(request.get_data(as_text=as_text)) 40 | return ret 41 | #except Exception e: 42 | #traceback.print_exc() 43 | #return str(e) 44 | 45 | if __name__ == '__main__': 46 | serve(app, host='0.0.0.0', port=5000) 47 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | waitress 3 | 4 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/template/python3-flask/template.yml: -------------------------------------------------------------------------------- 1 | language: python3-flask 2 | fprocess: python index.py 3 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-mapper.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | wc-mapper: 7 | lang: python3-flask 8 | handler: ./wc-mapper 9 | image: wc-mapper:latest 10 | environment: 11 | write_timeout: 1m 12 | read_timeout: 1m 13 | exec_timeout: 1m 14 | handler_wait_duration: 1m 15 | labels: 16 | "com.openfaas.scale.max": "5" 17 | "com.openfaas.scale.min": "5" 18 | limits: 19 | cpu: 2000m 20 | memory: 2048Mi 21 | requests: 22 | cpu: 2000m 23 | memory: 2048Mi 24 | constraints: 25 | - "kubernetes.io/hostname=kube-node-7" 26 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-mapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/wc-mapper/__init__.py -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-mapper/handler.py: -------------------------------------------------------------------------------- 1 | from minio import Minio 2 | import redis 3 | import time 4 | import json 5 | 6 | APP = "wc" 7 | minio_client = Minio("minio-service.yasb-mapreduce-db.svc.cluster.local:9000", access_key="admin123", secret_key="admin123", secure=False) 8 | redis_client = redis.Redis(host="redis.yasb-mapreduce-db.svc.cluster.local", port=6379) 9 | 10 | def handle(req): 11 | """handle a request to the function 12 | Args: 13 | req (str): request body 14 | """ 15 | event = json.loads(req) 16 | 17 | input_name = event["input_name"] 18 | input_part = int(event["input_part"]) 19 | reduce_num = int(event["reduce_num"]) 20 | 21 | read_start = time.time() 22 | 23 | input_object = minio_client.get_object(input_name, "part-%d" % input_part) 24 | input_data = input_object.data.decode("utf-8") 25 | print("------------------------------------------------------input_data_size%d" % len(input_data)) 26 | read_end = time.time() 27 | 28 | counts = {} 29 | 30 | lines = input_data.split("\n") 31 | 32 | print("------------------------------------------------------line_size%d" % len(lines)) 33 | for line in lines: 34 | words = line.strip().split(" ") 35 | for word in words: 36 | if word.isalpha(): 37 | if word not in counts: 38 | counts[word] = 0 39 | counts[word] = counts[word] + 1 40 | print("------------------------------------------------------count_size%d" % len(counts)) 41 | shuffle = {} 42 | for i in range(reduce_num): 43 | shuffle[i] = '' 44 | 45 | for word, count in counts.items(): 46 | reduce_id = hash(word) % reduce_num 47 | shuffle[reduce_id] = shuffle[reduce_id] + "%s:%d;" % (word, count) 48 | 49 | for i in range(reduce_num): 50 | if shuffle[i][-1] == ";": 51 | shuffle[i] = shuffle[i][:-1] 52 | 53 | com_end = time.time() 54 | 55 | for i in range(reduce_num): 56 | if not shuffle[i] == '': 57 | name = "%s:%s:%d:%d" % (input_name, APP, input_part, i) 58 | redis_client.set(name, shuffle[i]) 59 | 60 | store_end = time.time() 61 | 62 | result = { 63 | "read_start": read_start, 64 | "read_end": read_end, 65 | "com_end": com_end, 66 | "store_end": store_end 67 | } 68 | 69 | return json.dumps(result) 70 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-mapper/requirements.txt: -------------------------------------------------------------------------------- 1 | minio 2 | redis 3 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-reducer.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | wc-reducer: 7 | lang: python3-flask 8 | handler: ./wc-reducer 9 | image: wc-reducer:latest 10 | environment: 11 | write_timeout: 1m 12 | read_timeout: 1m 13 | exec_timeout: 1m 14 | handler_wait_duration: 1m 15 | labels: 16 | "com.openfaas.scale.max": "5" 17 | "com.openfaas.scale.min": "5" 18 | limits: 19 | cpu: 2000m 20 | memory: 2048Mi 21 | requests: 22 | cpu: 2000m 23 | memory: 2048Mi 24 | constraints: 25 | - "kubernetes.io/hostname=kube-node-7" 26 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-reducer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/functions/wc-reducer/__init__.py -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-reducer/handler.py: -------------------------------------------------------------------------------- 1 | import redis 2 | from minio import Minio 3 | import time 4 | import json 5 | import io 6 | 7 | APP = "wc" 8 | minio_client = Minio("minio-service.yasb-mapreduce-db.svc.cluster.local:9000", access_key="admin123", secret_key="admin123", secure=False) 9 | redis_client = redis.Redis(host="redis.yasb-mapreduce-db.svc.cluster.local", port=6379) 10 | 11 | def handle(req): 12 | """handle a request to the function 13 | Args: 14 | req (str): request body 15 | """ 16 | event = json.loads(req) 17 | 18 | input_name = event["input_name"] 19 | input_num = int(event["input_num"]) 20 | reduce_part = int(event["reduce_part"]) 21 | 22 | read_start = time.time() 23 | 24 | raw_datas = [] 25 | for i in range(input_num): 26 | name = "%s:%s:%d:%d" % (input_name, APP, i, reduce_part) 27 | if redis_client.exists(name): 28 | raw_data = redis_client.get(name) 29 | raw_datas.append(raw_data) 30 | 31 | read_end = time.time() 32 | 33 | counts = {} 34 | for raw_data in raw_datas: 35 | str_data = raw_data.decode("utf-8") 36 | pairs = [d.split(":") for d in str_data.split(";")] 37 | for pair in pairs: 38 | if pair[0] not in counts: 39 | counts[pair[0]] = 0 40 | counts[pair[0]] = counts[pair[0]] + int(pair[1]) 41 | 42 | com_end = time.time() 43 | 44 | output = "\n".join(["%s:%d" % (k, v) for k, v in counts.items()]) 45 | output_bucket = "%s-output" % APP 46 | 47 | if not minio_client.bucket_exists(output_bucket): 48 | minio_client.make_bucket(output_bucket) 49 | output_object = "part-%d" % reduce_part 50 | 51 | try: 52 | stat = minio_client.stat_object(output_bucket, output_object) 53 | minio_client.remove_object(output_bucket, output_object) 54 | except Exception as e: 55 | pass 56 | 57 | stream = io.BytesIO(output.encode()) 58 | stream_size = stream.getbuffer().nbytes 59 | minio_client.put_object(output_bucket, output_object, stream, length=stream_size) 60 | 61 | store_end = time.time() 62 | 63 | result = { 64 | "read_start": read_start, 65 | "read_end": read_end, 66 | "com_end": com_end, 67 | "store_end": store_end 68 | } 69 | 70 | return json.dumps(result) 71 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/functions/wc-reducer/requirements.txt: -------------------------------------------------------------------------------- 1 | redis 2 | minio 3 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # makefile for Text_datagen 3 | # 4 | # 5 | 6 | all: gen_random_text.cpp pgen_random_text.cpp 7 | g++ -Wall gen_random_text.cpp -I/usr/local/include/gsl -lgsl -lgslcblas -fexceptions -o gen_random_text 8 | g++ -Wall pgen_random_text.cpp -I/usr/local/include/gsl -lgsl -lgslcblas -fexceptions -o pgen_random_text 9 | 10 | .PHONY : clean 11 | clean : 12 | -rm gen_random_text 13 | -rm pgen_random_text 14 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/README: -------------------------------------------------------------------------------- 1 | Usage: gen_text_data.sh MODEL_NAME FIlE_NUM FILE_LINES LINE_WORDS OUT_DATA_DIR 2 | 3 | e.g., 500M = 8000 FIlE_NUM * 10000 LINE_WORDS 4 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/createBK.py: -------------------------------------------------------------------------------- 1 | from xmlrpc.client import ResponseError 2 | 3 | from minio import Minio 4 | 5 | minioClient = Minio("10.244.4.7:9000", access_key="admin123", secret_key="admin123", secure=False) 6 | print(111) 7 | 8 | try: 9 | minioClient.make_bucket("data-500m", location="us-east-123") 10 | except ResponseError as err: 11 | print(err) 12 | 13 | buckets = minioClient.list_buckets() 14 | for bucket in buckets: 15 | print(bucket.name, bucket.creation_date) 16 | # try: 17 | # minioClient.remove_bucket(bucket.name) 18 | # except ResponseError as err: 19 | # print(err) -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/gen_random_text: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/mapreduce/openfaas/wc/gen_data/gen_random_text -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/gen_text_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## 4 | # Function: Generate text data 5 | # 6 | # To Run: ./gen_text_data.sh lda_wiki1w 10 100 10000 gen_data/ 7 | # 8 | # Output in gen_data DIR 9 | # 10 | # By: Mingzijian, mingzijian@ict.ac.cn 11 | ## 12 | 13 | date 14 | 15 | # how much parameters at least 16 | PARA_LIMIT=4 17 | if [ "$#" -lt $PARA_LIMIT ]; then 18 | echo "Usage: $0" "MODEL_NAME" "FIlE_NUM" "FILE_LINES" "LINE_WORDS" "OUT_DATA_DIR" 19 | exit 1 20 | fi 21 | 22 | ## 23 | # main part of program 24 | ## 25 | MODEL_NAME=$1 26 | i=1 27 | #DATE_MARK=`date '+%Y-%m-%d_%H:%M:%S'` 28 | DATA_DIR=$5 29 | #"gen_data/$DATE_MARK" 30 | PARALLEL_NUM=21 31 | NUMBER_OF_FILES=$2 32 | LINES_PER_FILE=$3 33 | WORDS_PER_LINE=$4 34 | mkdir -p "$DATA_DIR" 35 | c=1 36 | while [ $i -le $NUMBER_OF_FILES ]; do 37 | ./gen_random_text $MODEL_NAME $LINES_PER_FILE $WORDS_PER_LINE $i > "$DATA_DIR/${MODEL_NAME}_${i}"& 38 | let i++ 39 | 40 | let c++ 41 | if [ $c -eq $PARALLEL_NUM ]; then 42 | c=1 43 | wait 44 | fi 45 | done 46 | wait 47 | 48 | date 49 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/lda_wiki1w/final.other: -------------------------------------------------------------------------------- 1 | num_topics 20 2 | num_terms 7762 3 | alpha 2.3817510310 4 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/removeBK.py: -------------------------------------------------------------------------------- 1 | from xmlrpc.client import ResponseError 2 | 3 | from minio import Minio 4 | 5 | minioClient = Minio("10.244.4.7:9000", access_key="admin123", secret_key="admin123", secure=False) 6 | print(111) 7 | 8 | # try: 9 | # minioClient.make_bucket("data-500m", location="us-east-123") 10 | # except ResponseError as err: 11 | # print(err) 12 | 13 | buckets = minioClient.list_buckets() 14 | for bucket in buckets: 15 | print(bucket.name, bucket.creation_date) 16 | try: 17 | minioClient.remove_bucket(bucket.name) 18 | except ResponseError as err: 19 | print(err) -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/showBK.py: -------------------------------------------------------------------------------- 1 | from xmlrpc.client import ResponseError 2 | 3 | from minio import Minio 4 | 5 | #minioClient = Minio("10.244.4.7:9000", access_key="admin123", secret_key="admin123", secure=False) 6 | minioClient = Minio("minio-service.yasb-mapreduce-db.svc.cluster.local:9000", access_key="admin123", secret_key="admin123", secure=False) 7 | print(111) 8 | 9 | # try: 10 | # minioClient.make_bucket("data-500m", location="us-east-123") 11 | # except ResponseError as err: 12 | # print(err) 13 | 14 | buckets = minioClient.list_buckets() 15 | print(len(buckets)) 16 | # for bucket in buckets: 17 | # print(bucket.name, bucket.creation_date) 18 | # try: 19 | # minioClient.remove_bucket(bucket.name) 20 | # except ResponseError as err: 21 | # print(err) -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/gen_data/upload.py: -------------------------------------------------------------------------------- 1 | from minio import Minio 2 | 3 | minioClient = Minio("10.244.4.7:9000", access_key="admin123", secret_key="admin123", secure=False) 4 | 5 | for i in range(5): 6 | local_path = "wc-100M-5/lda_wiki1w_%d" % (i+1) 7 | minioClient.fput_object("data-500m", object_name="part-%d" % i, file_path=local_path) 8 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/yaml/minio.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | # This name uniquely identifies the Deployment 5 | name: minio-deployment 6 | namespace: yasb-mapreduce-db 7 | spec: 8 | strategy: 9 | type: Recreate 10 | selector: 11 | matchLabels: 12 | app: minio 13 | template: 14 | metadata: 15 | labels: 16 | # Label is used as selector in the service. 17 | app: minio 18 | spec: 19 | volumes: 20 | - name: minio-volume 21 | hostPath: 22 | path: /mnt/minio 23 | type: DirectoryOrCreate 24 | nodeName: kube-node-7 25 | containers: 26 | - name: minio 27 | # Pulls the default MinIO image from Docker Hub 28 | image: minio/minio 29 | command: 30 | - /bin/bash 31 | - -c 32 | args: 33 | - minio server /data 34 | volumeMounts: 35 | - mountPath: /data 36 | name: minio-volume 37 | env: 38 | # MinIO access key and secret key 39 | - name: MINIO_ROOT_USER 40 | value: "admin123" 41 | - name: MINIO_ROOT_PASSWORD 42 | value: "admin123" 43 | ports: 44 | - containerPort: 9000 45 | --- 46 | apiVersion: v1 47 | kind: Service 48 | metadata: 49 | name: minio-service 50 | namespace: yasb-mapreduce-db 51 | spec: 52 | type: NodePort 53 | ports: 54 | - port: 9000 55 | targetPort: 9000 56 | protocol: TCP 57 | selector: 58 | app: minio 59 | -------------------------------------------------------------------------------- /mapreduce/openfaas/wc/yaml/redis.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: redis 6 | labels: 7 | project: mss 8 | app-name: redis 9 | namespace: yasb-mapreduce-db 10 | spec: 11 | type: NodePort 12 | ports: 13 | - port: 6379 14 | targetPort: 6379 15 | protocol: TCP 16 | #ports: 17 | #- port: 6379 18 | selector: 19 | project: mss 20 | app-name: redis 21 | --- 22 | apiVersion: apps/v1 23 | kind: Deployment 24 | metadata: 25 | name: redis 26 | labels: 27 | project: mss 28 | app-name: redis 29 | namespace: yasb-mapreduce-db 30 | spec: 31 | replicas: 1 32 | selector: 33 | matchLabels: 34 | project: mss 35 | app-name: redis 36 | template: 37 | metadata: 38 | labels: 39 | project: mss 40 | app-name: redis 41 | name: redis 42 | spec: 43 | nodeName: kube-node-7 44 | containers: 45 | - name: redis 46 | image: redis 47 | imagePullPolicy: IfNotPresent 48 | restartPolicy: Always 49 | 50 | -------------------------------------------------------------------------------- /mapreduce/ray/Workflow1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import subprocess 5 | import threading 6 | from flask import Flask 7 | 8 | app = Flask(__name__) 9 | 10 | mapper_name = "wc-mapper" 11 | reducer_name = "wc-reducer" 12 | mapper_num = 5 13 | reducer_num = 5 14 | headers = { 15 | "Content-Type": "application/json", 16 | "accept": "application/json", 17 | } 18 | mapurl = "http://10.244.0.158:8000/wc/map" 19 | reduceurl = "http://10.244.0.158:8000/wc/reduce" 20 | 21 | 22 | 23 | def post(url, data): 24 | text = requests.post(url, data=data, headers=headers).text 25 | 26 | 27 | def workflow(): 28 | start = time.time() 29 | 30 | 31 | threads = [] 32 | for i in range(mapper_num): 33 | req = json.dumps({"input_name": "data-500m", "input_part": i, "reduce_num": reducer_num}) 34 | t = threading.Thread(target=post, args=(mapurl, req)) 35 | threads.append(t) 36 | 37 | for t in threads: 38 | t.start() 39 | 40 | for t in threads: 41 | t.join() 42 | 43 | mapper_end = time.time() 44 | 45 | 46 | threads = [] 47 | for i in range(reducer_num): 48 | req = json.dumps({"input_name": "data-500m", "input_num": mapper_num, "reduce_part": i}) 49 | t = threading.Thread(target=post, args=(reduceurl, req)) 50 | threads.append(t) 51 | 52 | for t in threads: 53 | t.start() 54 | 55 | for t in threads: 56 | t.join() 57 | 58 | reducer_end = time.time() 59 | 60 | return "time: %d " % (reducer_end-start) 61 | 62 | 63 | 64 | 65 | @app.route('/hi') 66 | def index(): 67 | res = workflow() 68 | # get_res_info(reducer_res) 69 | return { 70 | "msg": "success", 71 | "data": res 72 | } 73 | 74 | 75 | if __name__ == "__main__": 76 | app.run() -------------------------------------------------------------------------------- /mapreduce/ray/mapreduce.yaml: -------------------------------------------------------------------------------- 1 | 2 | import_path: wc.main:ingress 3 | 4 | runtime_env: 5 | pip: 6 | - "redis" 7 | - "minio" 8 | - "numpy" 9 | 10 | 11 | 12 | host: 0.0.0.0 13 | 14 | port: 8000 15 | 16 | deployments: 17 | 18 | - name: MapService 19 | num_replicas: 5 20 | ray_actor_options: 21 | num_cpus: 1.0 22 | num_gpus: 0.0 23 | 24 | - name: ReduceService 25 | num_replicas: 5 26 | ray_actor_options: 27 | num_cpus: 1.0 28 | num_gpus: 0.0 29 | 30 | - name: WordCount 31 | -------------------------------------------------------------------------------- /mapreduce/ray/req.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | resp = requests.get("http://10.244.4.187:8000/ccc/test") 5 | print(resp.text) 6 | 7 | 8 | tt = json.dumps({"input_name": "data-500m", "input_part": 1, "reduce_num": 5}) 9 | resp2 = requests.post("http://10.244.4.187:8000/aaa/map", req=tt) 10 | 11 | print(resp2.text) 12 | -------------------------------------------------------------------------------- /mapreduce/ray/wc/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import ray 4 | import requests 5 | from fastapi import FastAPI 6 | from ray import serve 7 | from ray.serve.handle import RayServeDeploymentHandle 8 | from wc.service.map import MapService 9 | from wc.service.reduce import ReduceService 10 | from wc.model.post import PostBodyMap,PostBodyReduce,PostStr 11 | 12 | app = FastAPI() 13 | 14 | 15 | @serve.deployment() 16 | @serve.ingress(app) 17 | class WordCount: 18 | def __init__( 19 | self, 20 | map_handle: RayServeDeploymentHandle, 21 | reduce_handle: RayServeDeploymentHandle, 22 | ): 23 | self._map_handle = map_handle 24 | self._reduce_handle = reduce_handle 25 | 26 | @app.post("/wc/map") 27 | async def map(self, req: PostBodyMap): 28 | print("map") 29 | result = await self._map_handle.Mapper.remote(req) 30 | return {"message": 200} 31 | 32 | @app.post("/wc/reduce") 33 | async def reduce(self, req: PostBodyReduce): 34 | print("reduce") 35 | result = await self._reduce_handle.Reducer.remote(req) 36 | return {"message": 200} 37 | 38 | @app.get("/wc/test") 39 | async def root(self): 40 | return "Hello from 111!" 41 | 42 | map_handle = MapService.bind() 43 | reduce_handle = ReduceService.bind() 44 | 45 | ingress = WordCount.bind(map_handle,reduce_handle) 46 | 47 | -------------------------------------------------------------------------------- /mapreduce/ray/wc/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class PostStr(BaseModel): 5 | input_str: str 6 | 7 | class PostBodyMap(BaseModel): 8 | input_name: str 9 | input_part: int 10 | reduce_num: int 11 | 12 | 13 | 14 | class PostBodyReduce(BaseModel): 15 | input_name: str 16 | input_num: int 17 | reduce_part: int 18 | -------------------------------------------------------------------------------- /mapreduce/ray_stateful/Workflow1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import subprocess 5 | import threading 6 | from flask import Flask 7 | 8 | app = Flask(__name__) 9 | 10 | mapper_name = "wc-mapper" 11 | reducer_name = "wc-reducer" 12 | mapper_num = 5 13 | reducer_num = 5 14 | headers = { 15 | "Content-Type": "application/json", 16 | "accept": "application/json", 17 | } 18 | mapurl = "http://10.244.0.158:8000/wc/map" 19 | reduceurl = "http://10.244.0.158:8000/wc/reduce" 20 | 21 | 22 | 23 | def post(url, data): 24 | text = requests.post(url, data=data, headers=headers).text 25 | 26 | 27 | def workflow(): 28 | start = time.time() 29 | 30 | 31 | threads = [] 32 | for i in range(mapper_num): 33 | req = json.dumps({"input_name": "data-500m", "input_part": i, "reduce_num": reducer_num}) 34 | t = threading.Thread(target=post, args=(mapurl, req)) 35 | threads.append(t) 36 | 37 | for t in threads: 38 | t.start() 39 | 40 | for t in threads: 41 | t.join() 42 | 43 | mapper_end = time.time() 44 | 45 | 46 | threads = [] 47 | for i in range(reducer_num): 48 | req = json.dumps({"input_name": "data-500m", "input_num": mapper_num, "reduce_part": i}) 49 | t = threading.Thread(target=post, args=(reduceurl, req)) 50 | threads.append(t) 51 | 52 | for t in threads: 53 | t.start() 54 | 55 | for t in threads: 56 | t.join() 57 | 58 | reducer_end = time.time() 59 | 60 | return "time: %d " % (reducer_end-start) 61 | 62 | 63 | 64 | 65 | @app.route('/hi') 66 | def index(): 67 | res = workflow() 68 | # get_res_info(reducer_res) 69 | return { 70 | "msg": "success", 71 | "data": res 72 | } 73 | 74 | 75 | if __name__ == "__main__": 76 | app.run() -------------------------------------------------------------------------------- /mapreduce/ray_stateful/mapreduce.yaml: -------------------------------------------------------------------------------- 1 | 2 | import_path: wc.main:ingress 3 | 4 | runtime_env: 5 | pip: 6 | - "redis" 7 | - "minio" 8 | - "numpy" 9 | 10 | 11 | 12 | host: 0.0.0.0 13 | 14 | port: 8000 15 | 16 | deployments: 17 | 18 | - name: MapService 19 | num_replicas: 5 20 | ray_actor_options: 21 | num_cpus: 1.0 22 | num_gpus: 0.0 23 | 24 | - name: ReduceService 25 | num_replicas: 5 26 | ray_actor_options: 27 | num_cpus: 1.0 28 | num_gpus: 0.0 29 | 30 | - name: WordCount 31 | -------------------------------------------------------------------------------- /mapreduce/ray_stateful/req.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import ray 4 | # resp = requests.get("http://10.244.4.187:8000/ccc/test") 5 | # print(resp.text) 6 | 7 | 8 | tt = json.dumps({"input_name": "data-500m", "input_part": 1, "reduce_num": 5}) 9 | resp2 = requests.post("http://10.244.0.158:8000/wc/map", data=tt) 10 | 11 | print(resp2.text) 12 | print(ray.get(resp2.text)) 13 | -------------------------------------------------------------------------------- /mapreduce/ray_stateful/test.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | 4 | def saySorry(ls): 5 | print("1321？") 6 | print(ls) 7 | return "niho43w2" 8 | 9 | class MyThread(threading.Thread): 10 | 11 | def __init__(self, func, args=()): 12 | super(MyThread, self).__init__() 13 | self.func = func 14 | self.args = args 15 | 16 | def run(self): 17 | self.result = self.func(*self.args) 18 | 19 | def get_result(self): 20 | try: 21 | return self.result # 如果子线程不使用join方法，此处可能会报没有self.result的错误 22 | except Exception: 23 | return None 24 | 25 | 26 | if __name__ == '__main__': 27 | threads= [] 28 | for url in range(5): 29 | # t = threading.Thread(target=saySorry) 30 | t = threading.Thread(target=saySorry,args=("nini",)) 31 | threads.append(t) 32 | 33 | for t in threads: 34 | t.start() 35 | 36 | for t in threads: 37 | t.join() 38 | for t in threads: 39 | print(t.) -------------------------------------------------------------------------------- /mapreduce/ray_stateful/test1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | 4 | async def func1(num): 5 | print('--func1 start--') 6 | await asyncio.sleep(num) 7 | print('--func1 done--') 8 | return 'func1 ok' 9 | 10 | async def func2(num): 11 | print('--func2 start--') 12 | await asyncio.sleep(num) 13 | print('--func2 done--') 14 | return 'func2 ok' 15 | 16 | 17 | async def main(): 18 | task1 = asyncio.ensure_future(func1(3)) 19 | task2 = asyncio.ensure_future(func2(5)) 20 | tasks = [task1, task2] 21 | res = await asyncio.gather(*tasks) 22 | return res 23 | # done, pending = await asyncio.wait(tasks) 24 | # for t in done: 25 | # print(t.result()) 26 | # print(done) 27 | # print(pending) 28 | 29 | 30 | if __name__ == '__main__': 31 | loop = asyncio.get_event_loop() 32 | result = loop.run_until_complete(main()) 33 | print(result) 34 | -------------------------------------------------------------------------------- /mapreduce/ray_stateful/wc/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class PostStr(BaseModel): 5 | input_str: str 6 | 7 | class PostBodyMap(BaseModel): 8 | input_name: str 9 | input_part: int 10 | reduce_num: int 11 | 12 | 13 | 14 | class PostBodyReduce(BaseModel): 15 | input_name: str 16 | input_num: int 17 | reduce_part: int 18 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/compose-review.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | compose-review: 7 | lang: python3-flask 8 | handler: ./compose-review 9 | image: compose-review:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | constraints: 13 | - "kubernetes.io/hostname=kube-node-7" 14 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/compose-review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/compose-review/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/compose-review/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | arguments = ["title", "text", "username", "password", "rating"] 5 | 6 | def handle(req): 7 | """handle a request to the function 8 | Args: 9 | req (str): request body 10 | """ 11 | start = time.time() 12 | 13 | event = json.loads(req) 14 | 15 | response = {"time": {"compose-review": {"start_time": start}}} 16 | 17 | complete = False 18 | try: 19 | for arg in arguments: 20 | if event[arg] == '': 21 | break 22 | complete = True 23 | except Exception as e: 24 | pass 25 | 26 | if complete: 27 | response["body"] = event 28 | else: 29 | response["body"] = "Incomplete arguments" 30 | 31 | response["time"]["compose-review"]["end_time"] = time.time() 32 | return json.dumps(response) 33 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/compose-review/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/compose-review/requirements.txt -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/deploy_mr.sh: -------------------------------------------------------------------------------- 1 | funcs=("compose-review" "upload-user-id" "upload-movie-id" "mr-upload-text" "mr-upload-unique-id" "mr-compose-and-upload" "store-review" "upload-user-review" "upload-movie-review") 2 | for ((i=0;i<9;i++)) 3 | do 4 | faashwh-cli build -f ${funcs[$i]}.yml 5 | done 6 | 7 | 8 | for ((i=0;i<9;i++)) 9 | do 10 | faashwh-cli deploy -f ${funcs[$i]}.yml 11 | done 12 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/describe.sh: -------------------------------------------------------------------------------- 1 | funcs=("compose-review" "upload-user-id" "upload-movie-id" "mr-upload-text" "mr-upload-unique-id" "mr-compose-and-upload" "store-review" "upload-user-review" "upload-movie-review") 2 | for ((i=0;i<9;i++)) 3 | do 4 | faashwh-cli describe ${funcs[$i]} 5 | done 6 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-compose-and-upload.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | mr-compose-and-upload: 7 | lang: python3-flask 8 | handler: ./mr-compose-and-upload 9 | image: mr-compose-and-upload:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-compose-and-upload/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-compose-and-upload/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-compose-and-upload/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import json 3 | 4 | def handle(req): 5 | """handle a request to the function 6 | Args: 7 | req (str): request body 8 | """ 9 | start = time() 10 | 11 | events = json.loads(req) 12 | events = [json.loads(event) for event in events] 13 | 14 | body = {} 15 | response = {"time": {"mr-compose-and-upload": {"start_time": start}}} 16 | 17 | if len(events) < 4: 18 | body = 'Incomplete arguments' 19 | print("no") 20 | else: 21 | print("ok") 22 | try: 23 | for event in events: 24 | body.update(event["body"]) 25 | response["time"].update(event["time"]) 26 | body["timestamp"] = int(time()*1000) 27 | except Exception as e: 28 | body = 'Incomplete arguments' 29 | 30 | response["body"] = body 31 | response["time"]["mr-compose-and-upload"]["end_time"] = time() 32 | return json.dumps(response) 33 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-compose-and-upload/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-compose-and-upload/requirements.txt -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-db-op.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | mr-db-op: 7 | lang: python3-flask 8 | handler: ./mr-db-op 9 | image: mr-db-op:latest 10 | environment: 11 | write_timeout: 3m 12 | read_timeout: 3m 13 | exec_timeout: 3m 14 | handler_wait_duration: 3m 15 | labels: 16 | "com.openfaas.scale.max": "1" 17 | constraints: 18 | - "kubernetes.io/hostname=kube-node-7" 19 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-db-op/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-db-op/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-db-op/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==3.11.0 2 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-text.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | mr-upload-text: 7 | lang: python3-flask 8 | handler: ./mr-upload-text 9 | image: mr-upload-text:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-text/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-upload-text/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-text/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import json 3 | 4 | def handle(req): 5 | """handle a request to the function 6 | Args: 7 | req (str): request body 8 | """ 9 | start = time() 10 | 11 | event = json.loads(req) 12 | 13 | response = {"time": {"mr-upload-text": {"start_time": start}}} 14 | text = '' 15 | try: 16 | text = event["body"]["text"] 17 | response["time"].update(event["time"]) 18 | except Exception as e: 19 | response["body"] = 'Incomplete arguments' 20 | 21 | if text: 22 | response["body"] = {"text": text} 23 | 24 | response["time"]["mr-upload-text"]["end_time"] = time() 25 | 26 | return json.dumps(response) 27 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-text/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-upload-text/requirements.txt -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-unique-id.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | mr-upload-unique-id: 7 | lang: python3-flask 8 | handler: ./mr-upload-unique-id 9 | image: mr-upload-unique-id:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-unique-id/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-upload-unique-id/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-unique-id/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | from time import time 3 | import string 4 | import random 5 | 6 | def gen_random_digits(i): 7 | return "".join(random.sample(string.digits, i)) 8 | 9 | 10 | def handle(req): 11 | """handle a request to the function 12 | Args: 13 | req (str): request body 14 | """ 15 | start = time() 16 | 17 | event = json.loads(req) 18 | 19 | response = {"time": {"mr-upload-unique-id": {"start_time": start}}} 20 | 21 | machine_id = gen_random_digits(2) 22 | timestamp = str(int(time()*1000) - 1514764800000)[-11:] 23 | index_id = gen_random_digits(3) 24 | review_id = machine_id + timestamp + index_id 25 | 26 | response["body"] = {"review_id": review_id} 27 | response["time"]["mr-upload-unique-id"]["end_time"] = time() 28 | return json.dumps(response) 29 | 30 | 31 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/mr-upload-unique-id/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/mr-upload-unique-id/requirements.txt -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/rm_mr.sh: -------------------------------------------------------------------------------- 1 | funcs=("compose-review" "upload-user-id" "upload-movie-id" "mr-upload-text" "mr-upload-unique-id" "mr-compose-and-upload" "store-review" "upload-user-review" "upload-movie-review") 2 | for ((i=0;i<9;i++)) 3 | do 4 | faashwh-cli remove -f ${funcs[$i]}.yml 5 | kubectl delete svc ${funcs[$i]} -n openfaas-fn-hwh 6 | done 7 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/store-review.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | store-review: 7 | lang: python3-flask 8 | handler: ./store-review 9 | image: store-review:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/store-review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/store-review/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/store-review/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import pymongo 3 | import json 4 | 5 | """ 6 | myclient = pymongo.MongoClient("mongodb://review-storage-mongodb.movie-reviewing.svc.cluster.local:27017/", connect=False) 7 | mydb = myclient["review"] 8 | mycol = mydb["review"] 9 | """ 10 | arguments = set(["review_id", "timestamp", "user_id", "movie_id", "text", "rating"]) 11 | 12 | def handle(req): 13 | """handle a request to the function 14 | Args: 15 | req (str): request body 16 | """ 17 | start = time() 18 | 19 | myclient = pymongo.MongoClient("mongodb://review-storage-mongodb.movie-reviewing.svc.cluster.local:27017/") 20 | mydb = myclient["review"] 21 | mycol = mydb["review"] 22 | 23 | event = json.loads(req) 24 | response = {"time": {"store-review": {"start_time": start}}} 25 | 26 | if set(event["body"].keys()) == arguments: 27 | response["time"].update(event["time"]) 28 | mycol.insert_one(event["body"]) 29 | else: 30 | response['body'] = 'Incomplete arguments' 31 | 32 | response["time"]["store-review"]["end_time"] = time() 33 | return json.dumps(response) 34 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/store-review/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==3.11.0 2 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-id.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | upload-movie-id: 7 | lang: python3-flask 8 | handler: ./upload-movie-id 9 | image: upload-movie-id:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-id/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/upload-movie-id/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-id/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import pymongo 3 | import memcache 4 | import json 5 | import hashlib 6 | 7 | mongo_url = "mongodb://movie-id-mongodb.movie-reviewing.svc.cluster.local:27017/" 8 | memcache_url = "movie-id-memcached.movie-reviewing.svc.cluster.local:11211" 9 | 10 | mc = memcache.Client([memcache_url]) 11 | 12 | """ 13 | myclient = pymongo.MongoClient(mongo_url, connect=False) 14 | mydb = myclient['movie-id'] 15 | mycol = mydb["movie-id"] 16 | """ 17 | 18 | def hash_key(title): 19 | m = hashlib.md5() 20 | m.update(title.encode("utf-8")) 21 | return m.hexdigest() 22 | 23 | def handle(req): 24 | """handle a request to the function 25 | Args: 26 | req (str): request body 27 | """ 28 | start = time() 29 | 30 | myclient = pymongo.MongoClient(mongo_url) 31 | mydb = myclient['movie-id'] 32 | mycol = mydb["movie-id"] 33 | 34 | event = json.loads(req) 35 | body = '' 36 | title = '' 37 | title_hash = '' 38 | rating = -1 39 | response = {"time": {"upload-movie-id": {"start_time": start}}} 40 | try: 41 | title = event["body"]["title"] 42 | title_hash = hash_key(title) 43 | rating = int(event["body"]["rating"]) 44 | except Exception as e: 45 | response['body'] = 'Incomplete arguments' 46 | 47 | if title and rating > -1: 48 | movie_id = '' 49 | mmc_movie_id = mc.get(title_hash) 50 | if mmc_movie_id != None: 51 | movie_id = mmc_movie_id 52 | response["body"] = {"movie_id": movie_id, 'rating': rating} 53 | else: 54 | myquery = { "title": title } 55 | mydoc = mycol.find(myquery) 56 | if mydoc.count() > 0: 57 | it = mydoc.next() 58 | movie_id = it["movie_id"] 59 | response["body"] = {"movie_id": movie_id, 'rating': rating} 60 | mc.set(title_hash, movie_id) 61 | else: 62 | response["body"] = 'No movie ' + title 63 | 64 | response["time"]["upload-movie-id"]["end_time"] = time() 65 | return json.dumps(response) 66 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-id/requirements.txt: -------------------------------------------------------------------------------- 1 | python-memcached 2 | pymongo==3.11.0 3 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-review.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | upload-movie-review: 7 | lang: python3-flask 8 | handler: ./upload-movie-review 9 | image: upload-movie-review:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/upload-movie-review/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-review/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import pymongo 3 | import json 4 | 5 | """ 6 | myclient = pymongo.MongoClient("mongodb://movie-review-mongodb.movie-reviewing.svc.cluster.local:27017/", connect=False) 7 | mydb = myclient["movie-review"] 8 | mycol = mydb["movie-review"] 9 | """ 10 | 11 | def handle(req): 12 | """handle a request to the function 13 | Args: 14 | req (str): request body 15 | """ 16 | start = time() 17 | 18 | myclient = pymongo.MongoClient("mongodb://movie-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 19 | mydb = myclient["movie-review"] 20 | mycol = mydb["movie-review"] 21 | 22 | event = json.loads(req) 23 | response = {"time": {"upload-movie-review": {"start_time": start}}} 24 | 25 | try: 26 | movie_id = event["body"]["movie_id"] 27 | review_id = event["body"]["review_id"] 28 | timestamp = event["body"]["timestamp"] 29 | 30 | myquery = {"movie_id": movie_id} 31 | mydoc = mycol.find(myquery) 32 | if mydoc.count() > 0: 33 | reviews = json.loads(mydoc.next()["reviews"]) 34 | reviews.append((review_id, timestamp)) 35 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 36 | mycol.update_one(myquery, reviews_update) 37 | else: 38 | body = {"movie_id": movie_id, "reviews": json.dumps([(review_id, timestamp)])} 39 | mycol.insert_one(body) 40 | except Exception as e: 41 | response['body'] = 'Incomplete arguments' 42 | 43 | response["time"]["upload-movie-review"]["end_time"] = time() 44 | return json.dumps(response) 45 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-movie-review/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==3.11.0 2 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-id.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | upload-user-id: 7 | lang: python3-flask 8 | handler: ./upload-user-id 9 | image: upload-user-id:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | constraints: 13 | - "kubernetes.io/hostname=kube-node-7" 14 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-id/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/upload-user-id/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-id/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import pymongo 3 | import memcache 4 | import json 5 | 6 | mongo_url = "mongodb://user-mongodb.movie-reviewing.svc.cluster.local:27017/" 7 | memcache_url = "user-memcached.movie-reviewing.svc.cluster.local:11211" 8 | 9 | mc = memcache.Client([memcache_url]) 10 | """ 11 | myclient = pymongo.MongoClient(mongo_url, connect=False) 12 | mydb = myclient['user'] 13 | mycol = mydb["user"] 14 | """ 15 | 16 | def handle(req): 17 | """handle a request to the function 18 | Args: 19 | req (str): request body 20 | """ 21 | start = time() 22 | 23 | myclient = pymongo.MongoClient(mongo_url) 24 | mydb = myclient['user'] 25 | mycol = mydb["user"] 26 | 27 | event = json.loads(req) 28 | body = '' 29 | username = '' 30 | response = {"time": {"upload-user-id": {"start_time": start}}} 31 | try: 32 | username = event["body"]["username"] 33 | except Exception as e: 34 | body = 'Incomplete arguments' 35 | 36 | if username: 37 | user_id = -1 38 | mmc_user_id = mc.get(username+":user_id") 39 | if mmc_user_id != None: 40 | user_id = mmc_user_id 41 | body = {"user_id": user_id} 42 | else: 43 | myquery = { "username": username } 44 | mydoc = mycol.find(myquery) 45 | print(mydoc) 46 | if mydoc.count() > 0: 47 | it = mydoc.next() 48 | user_id = it["user_id"] 49 | body = {"user_id": user_id} 50 | mc.set(username+":user_id", user_id) 51 | else: 52 | body = 'No user ' + username 53 | 54 | response["body"] = body 55 | 56 | response["time"]["upload-user-id"]["end_time"] = time() 57 | return json.dumps(response) 58 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-id/requirements.txt: -------------------------------------------------------------------------------- 1 | python-memcached 2 | pymongo==3.11.0 3 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-review.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31119 5 | functions: 6 | upload-user-review: 7 | lang: python3-flask 8 | handler: ./upload-user-review 9 | image: upload-user-review:latest 10 | labels: 11 | "com.openfaas.scale.max": "1" 12 | "com.openfaas.scale.min": "1" 13 | constraints: 14 | - "kubernetes.io/hostname=kube-node-7" 15 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-review/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/media_service/openfaas/mr/functions/upload-user-review/__init__.py -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-review/handler.py: -------------------------------------------------------------------------------- 1 | from time import time 2 | import pymongo 3 | import json 4 | 5 | """ 6 | myclient = pymongo.MongoClient("mongodb://user-review-mongodb.movie-reviewing.svc.cluster.local:27017/", connect=False) 7 | mydb = myclient["user-review"] 8 | mycol = mydb["user-review"] 9 | """ 10 | 11 | def handle(req): 12 | """handle a request to the function 13 | Args: 14 | req (str): request body 15 | """ 16 | start = time() 17 | 18 | myclient = pymongo.MongoClient("mongodb://user-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 19 | mydb = myclient["user-review"] 20 | mycol = mydb["user-review"] 21 | 22 | event = json.loads(req) 23 | response = {"time": {"upload-user-review": {"start_time": start}}} 24 | 25 | try: 26 | user_id = event["body"]["user_id"] 27 | review_id = event["body"]["review_id"] 28 | timestamp = event["body"]["timestamp"] 29 | 30 | myquery = {"user_id": user_id} 31 | mydoc = mycol.find(myquery) 32 | if mydoc.count() > 0: 33 | reviews = json.loads(mydoc.next()["reviews"]) 34 | reviews.append((review_id, timestamp)) 35 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 36 | mycol.update_one(myquery, reviews_update) 37 | else: 38 | body = {"user_id": user_id, "reviews": json.dumps([(review_id, timestamp)])} 39 | mycol.insert_one(body) 40 | except Exception as e: 41 | response['body'] = 'Incomplete arguments' 42 | 43 | response["time"]["upload-user-review"]["end_time"] = time() 44 | return json.dumps(response) 45 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/functions/upload-user-review/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==3.11.0 2 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/namespaces.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: movie-reviewing 5 | 6 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/cast-info-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: cast-info-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: cast-info-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: cast-info-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: cast-info-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: cast-info-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: cast-info-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: cast-info-mongodb 36 | name: cast-info-mongodb 37 | spec: 38 | containers: 39 | - name: cast-info-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/movie-id-memcached.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: movie-id-memcached 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: movie-id-memcached 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 11211 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: movie-id-memcached 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: movie-id-memcached 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: movie-id-memcached 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: movie-id-memcached 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: movie-id-memcached 36 | name: movie-id-memcached 37 | spec: 38 | containers: 39 | - name: movie-id-memcached 40 | image: memcached 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/movie-id-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: movie-id-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: movie-id-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: movie-id-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: movie-id-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: movie-id-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: movie-id-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: movie-id-mongodb 36 | name: movie-id-mongodb 37 | spec: 38 | containers: 39 | - name: movie-id-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/movie-info-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: movie-info-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: movie-info-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: movie-info-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: movie-info-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: movie-info-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: movie-info-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: movie-info-mongodb 36 | name: movie-info-mongodb 37 | spec: 38 | containers: 39 | - name: movie-info-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/movie-review-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: movie-review-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: movie-review-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: movie-review-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: movie-review-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: movie-review-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: movie-review-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: movie-review-mongodb 36 | name: movie-review-mongodb 37 | spec: 38 | containers: 39 | - name: movie-review-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/plot-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: plot-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: plot-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: plot-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: plot-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: plot-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: plot-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: plot-mongodb 36 | name: plot-mongodb 37 | spec: 38 | containers: 39 | - name: plot-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/review-storage-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: review-storage-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: review-storage-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: review-storage-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: review-storage-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: review-storage-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: review-storage-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: review-storage-mongodb 36 | name: review-storage-mongodb 37 | spec: 38 | containers: 39 | - name: review-storage-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/user-memcached.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: user-memcached 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: user-memcached 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 11211 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: user-memcached 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: user-memcached 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: user-memcached 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: user-memcached 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: user-memcached 36 | name: user-memcached 37 | spec: 38 | containers: 39 | - name: user-memcached 40 | image: memcached 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/user-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: user-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: user-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: user-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: user-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: user-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: user-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: user-mongodb 36 | name: user-mongodb 37 | spec: 38 | containers: 39 | - name: user-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/openfaas/mr/yaml/yaml-db/user-review-mongodb.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: user-review-mongodb 6 | labels: 7 | death-star-project: movie-reviewing 8 | app-name: user-review-mongodb 9 | namespace: movie-reviewing 10 | spec: 11 | ports: 12 | - port: 27017 13 | selector: 14 | death-star-project: movie-reviewing 15 | app-name: user-review-mongodb 16 | --- 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: user-review-mongodb 21 | labels: 22 | death-star-project: movie-reviewing 23 | app-name: user-review-mongodb 24 | namespace: movie-reviewing 25 | spec: 26 | replicas: 1 27 | selector: 28 | matchLabels: 29 | death-star-project: movie-reviewing 30 | app-name: user-review-mongodb 31 | template: 32 | metadata: 33 | labels: 34 | death-star-project: movie-reviewing 35 | app-name: user-review-mongodb 36 | name: user-review-mongodb 37 | spec: 38 | containers: 39 | - name: user-review-mongodb 40 | image: mongo 41 | imagePullPolicy: IfNotPresent 42 | restartPolicy: Always 43 | 44 | -------------------------------------------------------------------------------- /media_service/ray/ms/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | from typing import List, Set, Union 4 | 5 | 6 | 7 | 8 | class PostItem(BaseModel): 9 | portfolioType: str 10 | portfolio: str 11 | 12 | 13 | class PostStr(BaseModel): 14 | input_str: str 15 | 16 | 17 | class PostBodyMap(BaseModel): 18 | input_name: str 19 | input_part: int 20 | reduce_num: int 21 | 22 | 23 | class PostBodyReduce(BaseModel): 24 | input_name: str 25 | input_num: int 26 | reduce_part: int 27 | -------------------------------------------------------------------------------- /media_service/ray/ms/requirements/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==4.3.3 2 | pymemcache==4.0.0 -------------------------------------------------------------------------------- /media_service/ray/ms/service/compose_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import time 4 | import json 5 | 6 | from ms.model.post import PostItem 7 | from fastapi import Body 8 | 9 | arguments = ["title", "text", "username", "password", "rating"] 10 | 11 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 12 | class ComposeReviewService(object): 13 | # def __init__(self): 14 | def ComposeReview(self, body: Dict): 15 | print(123) 16 | try: 17 | start = time.time() 18 | event = body 19 | response = {"time": {"compose-review": {"start_time": start}}} 20 | 21 | complete = False 22 | try: 23 | for arg in arguments: 24 | if event[arg] == '': 25 | break 26 | complete = True 27 | except Exception as e: 28 | pass 29 | 30 | if complete: 31 | response["body"] = event 32 | else: 33 | response["body"] = "Incomplete arguments" 34 | 35 | response["time"]["compose-review"]["end_time"] = time.time() 36 | print(response) 37 | except Exception as e: 38 | print(e) 39 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 40 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 41 | else: 42 | print("success") 43 | 44 | 45 | return response 46 | 47 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/mr_compose_and_upload.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | from time import time 5 | 6 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 7 | class MrComposeAndUploadService(object): 8 | # def __init__(self): 9 | def MrComposeAndUpload(self, body1: List): 10 | print(123) 11 | try: 12 | start = time() 13 | events = body1 14 | body = {} 15 | response = {"time": {"mr-compose-and-upload": {"start_time": start}}} 16 | 17 | if len(events) < 4: 18 | body = 'Incomplete arguments' 19 | else: 20 | try: 21 | for event in events: 22 | body.update(event["body"]) 23 | response["time"].update(event["time"]) 24 | body["timestamp"] = int(time() * 1000) 25 | except Exception as e: 26 | body = 'Incomplete arguments' 27 | 28 | response["body"] = body 29 | response["time"]["mr-compose-and-upload"]["end_time"] = time() 30 | print(response) 31 | except Exception as e: 32 | print(e) 33 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 34 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 35 | else: 36 | print("success") 37 | 38 | 39 | return response 40 | 41 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/mr_upload_text.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | from time import time 5 | 6 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 7 | class MrUploadTextService(object): 8 | # def __init__(self): 9 | def MrUploadText(self, body1: Dict): 10 | print(123) 11 | try: 12 | start = time() 13 | 14 | event = body1 15 | 16 | response = {"time": {"mr-upload-text": {"start_time": start}}} 17 | text = '' 18 | try: 19 | text = event["body"]["text"] 20 | response["time"].update(event["time"]) 21 | except Exception as e: 22 | response["body"] = 'Incomplete arguments' 23 | 24 | if text: 25 | response["body"] = {"text": text} 26 | 27 | response["time"]["mr-upload-text"]["end_time"] = time() 28 | print(response) 29 | except Exception as e: 30 | print(e) 31 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 32 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 33 | else: 34 | print("success") 35 | 36 | 37 | return response 38 | 39 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/mr_upload_unique_id.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | from time import time 4 | import string 5 | import random 6 | 7 | def gen_random_digits(i): 8 | return "".join(random.sample(string.digits, i)) 9 | 10 | 11 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 12 | class MrUploadUniqueIdService(object): 13 | # def __init__(self): 14 | def MrUploadUniqueId(self, body1: Dict): 15 | print(123) 16 | try: 17 | start = time() 18 | 19 | event = body1 20 | 21 | response = {"time": {"mr-upload-unique-id": {"start_time": start}}} 22 | 23 | machine_id = gen_random_digits(2) 24 | timestamp = str(int(time() * 1000) - 1514764800000)[-11:] 25 | index_id = gen_random_digits(3) 26 | review_id = machine_id + timestamp + index_id 27 | 28 | response["body"] = {"review_id": review_id} 29 | response["time"]["mr-upload-unique-id"]["end_time"] = time() 30 | print(response) 31 | except Exception as e: 32 | print(e) 33 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 34 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 35 | else: 36 | print("success") 37 | 38 | 39 | return response 40 | 41 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/store_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | 9 | 10 | arguments = set(["review_id", "timestamp", "user_id", "movie_id", "text", "rating"]) 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class StoreReviewService(object): 14 | # def __init__(self): 15 | def StoreReview(self, body1: Dict): 16 | print(123) 17 | try: 18 | start = time() 19 | 20 | myclient = pymongo.MongoClient("mongodb://review-storage-mongodb.movie-reviewing.svc.cluster.local:27017/") 21 | mydb = myclient["review"] 22 | mycol = mydb["review"] 23 | 24 | event = body1 25 | response = {"time": {"store-review": {"start_time": start}}} 26 | 27 | if set(event["body"].keys()) == arguments: 28 | response["time"].update(event["time"]) 29 | mycol.insert_one(event["body"]) 30 | else: 31 | response['body'] = 'Incomplete arguments' 32 | 33 | response["time"]["store-review"]["end_time"] = time() 34 | print(response) 35 | except Exception as e: 36 | print(e) 37 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 38 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 39 | else: 40 | print("success") 41 | 42 | 43 | return response 44 | 45 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/upload_movie_id.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | from time import time 5 | import pymongo 6 | from pymemcache.client.base import Client 7 | import json 8 | import hashlib 9 | 10 | mongo_url = "mongodb://movie-id-mongodb.movie-reviewing.svc.cluster.local:27017/" 11 | memcache_url = "movie-id-memcached.movie-reviewing.svc.cluster.local:11211" 12 | 13 | mc = Client(memcache_url) 14 | 15 | def hash_key(title): 16 | m = hashlib.md5() 17 | m.update(title.encode("utf-8")) 18 | return m.hexdigest() 19 | 20 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 21 | class UploadMovieIdService(object): 22 | # def __init__(self): 23 | def UploadMovieId(self, body1: Dict): 24 | print(123) 25 | try: 26 | start = time() 27 | 28 | myclient = pymongo.MongoClient(mongo_url) 29 | mydb = myclient['movie-id'] 30 | mycol = mydb["movie-id"] 31 | 32 | event = body1 33 | body = '' 34 | title = '' 35 | title_hash = '' 36 | rating = -1 37 | response = {"time": {"upload-movie-id": {"start_time": start}}} 38 | try: 39 | title = event["body"]["title"] 40 | title_hash = hash_key(title) 41 | rating = int(event["body"]["rating"]) 42 | except Exception as e: 43 | response['body'] = 'Incomplete arguments' 44 | 45 | if title and rating > -1: 46 | movie_id = '' 47 | mmc_movie_id = mc.get(title_hash) 48 | if mmc_movie_id != None: 49 | movie_id = mmc_movie_id 50 | response["body"] = {"movie_id": movie_id, 'rating': rating} 51 | else: 52 | myquery = {"title": title} 53 | mydoc = mycol.find(myquery) 54 | if mydoc.count() > 0: 55 | it = mydoc.next() 56 | movie_id = it["movie_id"] 57 | response["body"] = {"movie_id": movie_id, 'rating': rating} 58 | mc.set(title_hash, movie_id) 59 | else: 60 | response["body"] = 'No movie ' + title 61 | 62 | response["time"]["upload-movie-id"]["end_time"] = time() 63 | 64 | except Exception as e: 65 | print(e) 66 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 67 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 68 | else: 69 | print("success") 70 | 71 | 72 | return response 73 | 74 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/upload_movie_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | 9 | 10 | 11 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 12 | class UploadMovieReviewService(object): 13 | # def __init__(self): 14 | def UploadMovieReview(self, body1: Dict): 15 | print(123) 16 | try: 17 | start = time() 18 | 19 | myclient = pymongo.MongoClient("mongodb://movie-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 20 | mydb = myclient["movie-review"] 21 | mycol = mydb["movie-review"] 22 | 23 | event = body1 24 | response = {"time": {"upload-movie-review": {"start_time": start}}} 25 | 26 | try: 27 | movie_id = event["body"]["movie_id"] 28 | review_id = event["body"]["review_id"] 29 | timestamp = event["body"]["timestamp"] 30 | 31 | myquery = {"movie_id": movie_id} 32 | mydoc = mycol.find(myquery) 33 | if mydoc.count() > 0: 34 | reviews = json.loads(mydoc.next()["reviews"]) 35 | reviews.append((review_id, timestamp)) 36 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 37 | mycol.update_one(myquery, reviews_update) 38 | else: 39 | body = {"movie_id": movie_id, "reviews": json.dumps([(review_id, timestamp)])} 40 | mycol.insert_one(body) 41 | except Exception as e: 42 | response['body'] = 'Incomplete arguments' 43 | 44 | response["time"]["upload-movie-review"]["end_time"] = time() 45 | print(response) 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | 54 | return response 55 | 56 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/upload_user_id.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | from time import time 5 | import pymongo 6 | from pymemcache.client.base import Client 7 | import json 8 | 9 | mongo_url = "mongodb://user-mongodb.movie-reviewing.svc.cluster.local:27017/" 10 | memcache_url = "user-memcached.movie-reviewing.svc.cluster.local:11211" 11 | 12 | mc = Client(memcache_url) 13 | 14 | arguments = ["title", "text", "username", "password", "rating"] 15 | 16 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 17 | class UploadUserIdService(object): 18 | # def __init__(self): 19 | def UploadUserId(self, body1: Dict): 20 | print(123) 21 | try: 22 | start = time() 23 | 24 | myclient = pymongo.MongoClient(mongo_url) 25 | mydb = myclient['user'] 26 | mycol = mydb["user"] 27 | 28 | event = body1 29 | body = '' 30 | username = '' 31 | response = {"time": {"upload-user-id": {"start_time": start}}} 32 | try: 33 | username = event["body"]["username"] 34 | except Exception as e: 35 | body = 'Incomplete arguments' 36 | 37 | if username: 38 | user_id = -1 39 | mmc_user_id = mc.get(username + ":user_id") 40 | if mmc_user_id != None: 41 | user_id = mmc_user_id 42 | body = {"user_id": user_id} 43 | else: 44 | myquery = {"username": username} 45 | mydoc = mycol.find(myquery) 46 | if mydoc.count() > 0: 47 | it = mydoc.next() 48 | user_id = it["user_id"] 49 | body = {"user_id": user_id} 50 | mc.set(username + ":user_id", user_id) 51 | else: 52 | body = 'No user ' + username 53 | 54 | response["body"] = body 55 | 56 | response["time"]["upload-user-id"]["end_time"] = time() 57 | print(response) 58 | except Exception as e: 59 | print(e) 60 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 61 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 62 | else: 63 | print("success") 64 | 65 | 66 | return response 67 | 68 | -------------------------------------------------------------------------------- /media_service/ray/ms/service/upload_user_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | 9 | 10 | 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class UploadUserReviewService(object): 14 | # def __init__(self): 15 | def UploadUserReview(self, body1: Dict): 16 | print(123) 17 | try: 18 | start = time() 19 | 20 | myclient = pymongo.MongoClient("mongodb://user-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 21 | mydb = myclient["user-review"] 22 | mycol = mydb["user-review"] 23 | 24 | event = body1 25 | response = {"time": {"upload-user-review": {"start_time": start}}} 26 | 27 | try: 28 | user_id = event["body"]["user_id"] 29 | review_id = event["body"]["review_id"] 30 | timestamp = event["body"]["timestamp"] 31 | 32 | myquery = {"user_id": user_id} 33 | mydoc = mycol.find(myquery) 34 | if mydoc.count() > 0: 35 | reviews = json.loads(mydoc.next()["reviews"]) 36 | reviews.append((review_id, timestamp)) 37 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 38 | mycol.update_one(myquery, reviews_update) 39 | else: 40 | body = {"user_id": user_id, "reviews": json.dumps([(review_id, timestamp)])} 41 | mycol.insert_one(body) 42 | except Exception as e: 43 | response['body'] = 'Incomplete arguments' 44 | 45 | response["time"]["upload-user-review"]["end_time"] = time() 46 | except Exception as e: 47 | print(e) 48 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 49 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 50 | else: 51 | print("success") 52 | 53 | 54 | return response 55 | 56 | -------------------------------------------------------------------------------- /media_service/ray_stateful/media.yaml: -------------------------------------------------------------------------------- 1 | import_path: so.main:ingress 2 | 3 | runtime_env: 4 | pip: 5 | - "pymongo==4.3.3" 6 | - "pymemcache==4.0.0" 7 | 8 | host: 0.0.0.0 9 | 10 | port: 8000 11 | 12 | deployments: 13 | 14 | - name: ComposeReviewService 15 | num_replicas: 1 16 | ray_actor_options: 17 | num_cpus: 1.0 18 | num_gpus: 0.0 19 | 20 | - name: UploadUserIdService 21 | num_replicas: 1 22 | ray_actor_options: 23 | num_cpus: 1.0 24 | num_gpus: 0.0 25 | 26 | - name: UploadMovieIdService 27 | num_replicas: 1 28 | ray_actor_options: 29 | num_cpus: 1.0 30 | num_gpus: 0.0 31 | 32 | - name: MrUploadTextService 33 | num_replicas: 1 34 | ray_actor_options: 35 | num_cpus: 1.0 36 | num_gpus: 0.0 37 | 38 | - name: MrUploadUniqueIdService 39 | num_replicas: 1 40 | ray_actor_options: 41 | num_cpus: 1.0 42 | num_gpus: 0.0 43 | 44 | 45 | - name: MrComposeAndUploadService 46 | num_replicas: 1 47 | ray_actor_options: 48 | num_cpus: 1.0 49 | num_gpus: 0.0 50 | 51 | 52 | 53 | - name: StoreReviewService 54 | num_replicas: 1 55 | ray_actor_options: 56 | num_cpus: 1.0 57 | num_gpus: 0.0 58 | 59 | 60 | - name: UploadUserReviewService 61 | num_replicas: 1 62 | ray_actor_options: 63 | num_cpus: 1.0 64 | num_gpus: 0.0 65 | 66 | 67 | - name: UploadMovieReviewService 68 | num_replicas: 1 69 | ray_actor_options: 70 | num_cpus: 1.0 71 | num_gpus: 0.0 72 | 73 | 74 | - name: MediaService -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | from typing import List, Set, Union 4 | 5 | 6 | 7 | 8 | class PostItem(BaseModel): 9 | portfolioType: str 10 | portfolio: str 11 | 12 | 13 | class PostStr(BaseModel): 14 | input_str: str 15 | 16 | 17 | class PostBodyMap(BaseModel): 18 | input_name: str 19 | input_part: int 20 | reduce_num: int 21 | 22 | 23 | class PostBodyReduce(BaseModel): 24 | input_name: str 25 | input_num: int 26 | reduce_part: int 27 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/requirements/requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==4.3.3 2 | pymemcache==4.0.0 -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/compose_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import time 4 | import json 5 | import ray 6 | 7 | from ms.model.post import PostItem 8 | from fastapi import Body 9 | 10 | arguments = ["title", "text", "username", "password", "rating"] 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class ComposeReviewService(object): 14 | # def __init__(self): 15 | def ComposeReview(self, body: Dict): 16 | print(123) 17 | try: 18 | start = time.time() 19 | event = body 20 | response = {"time": {"compose-review": {"start_time": start}}} 21 | 22 | complete = False 23 | try: 24 | for arg in arguments: 25 | if event[arg] == '': 26 | break 27 | complete = True 28 | except Exception as e: 29 | pass 30 | 31 | if complete: 32 | response["body"] = event 33 | else: 34 | response["body"] = "Incomplete arguments" 35 | 36 | response["time"]["compose-review"]["end_time"] = time.time() 37 | print(response) 38 | resp = ray.put(response) 39 | except Exception as e: 40 | print(e) 41 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 42 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 43 | else: 44 | print("success") 45 | 46 | 47 | return resp 48 | 49 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/mr_compose_and_upload.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import ray 4 | from time import time 5 | 6 | 7 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 8 | class MrComposeAndUploadService(object): 9 | # def __init__(self): 10 | def MrComposeAndUpload(self, body1: List): 11 | print(123) 12 | try: 13 | start = time() 14 | print("body1:") 15 | print(body1) 16 | events = body1 17 | print(len(events)) 18 | body = {} 19 | response = {"time": {"mr-compose-and-upload": {"start_time": start}}} 20 | 21 | if len(events) < 4: 22 | body = 'Incomplete arguments' 23 | else: 24 | for ev in events: 25 | event = ray.get(ray.get(ev)) 26 | body.update(event["body"]) 27 | response["time"].update(event["time"]) 28 | body["timestamp"] = int(time() * 1000) 29 | # body = 'Incomplete arguments' 30 | 31 | response["body"] = body 32 | response["time"]["mr-compose-and-upload"]["end_time"] = time() 33 | print(response) 34 | except Exception as e: 35 | print(e) 36 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 37 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 38 | else: 39 | print("success") 40 | 41 | return ray.put(response) 42 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/mr_upload_text.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import ray 4 | from time import time 5 | 6 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 7 | class MrUploadTextService(object): 8 | # def __init__(self): 9 | def MrUploadText(self, body1: Dict): 10 | print(123) 11 | try: 12 | start = time() 13 | 14 | event = body1 15 | 16 | response = {"time": {"mr-upload-text": {"start_time": start}}} 17 | text = '' 18 | try: 19 | text = event["body"]["text"] 20 | response["time"].update(event["time"]) 21 | except Exception as e: 22 | response["body"] = 'Incomplete arguments' 23 | 24 | if text: 25 | response["body"] = {"text": text} 26 | 27 | response["time"]["mr-upload-text"]["end_time"] = time() 28 | print(response) 29 | 30 | except Exception as e: 31 | print(e) 32 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 33 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 34 | else: 35 | print("success") 36 | 37 | 38 | return ray.put(response) 39 | 40 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/mr_upload_unique_id.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | from time import time 4 | import string 5 | import random 6 | import ray 7 | 8 | def gen_random_digits(i): 9 | return "".join(random.sample(string.digits, i)) 10 | 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class MrUploadUniqueIdService(object): 14 | # def __init__(self): 15 | def MrUploadUniqueId(self, body1: Dict): 16 | print(123) 17 | try: 18 | start = time() 19 | 20 | event = body1 21 | 22 | response = {"time": {"mr-upload-unique-id": {"start_time": start}}} 23 | 24 | machine_id = gen_random_digits(2) 25 | timestamp = str(int(time() * 1000) - 1514764800000)[-11:] 26 | index_id = gen_random_digits(3) 27 | review_id = machine_id + timestamp + index_id 28 | 29 | response["body"] = {"review_id": review_id} 30 | response["time"]["mr-upload-unique-id"]["end_time"] = time() 31 | print(response) 32 | 33 | except Exception as e: 34 | print(e) 35 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 36 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 37 | else: 38 | print("success") 39 | 40 | 41 | return ray.put(response) 42 | 43 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/store_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | import ray 9 | 10 | arguments = set(["review_id", "timestamp", "user_id", "movie_id", "text", "rating"]) 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class StoreReviewService(object): 14 | # def __init__(self): 15 | def StoreReview(self, body1: Dict): 16 | print(123) 17 | try: 18 | start = time() 19 | 20 | myclient = pymongo.MongoClient("mongodb://review-storage-mongodb.movie-reviewing.svc.cluster.local:27017/") 21 | mydb = myclient["review"] 22 | mycol = mydb["review"] 23 | print("body1:") 24 | print(body1) 25 | event = body1 26 | print(event) 27 | response = {"time": {"store-review": {"start_time": start}}} 28 | 29 | if set(event["body"].keys()) == arguments: 30 | response["time"].update(event["time"]) 31 | mycol.insert_one(event["body"]) 32 | else: 33 | response['body'] = 'Incomplete arguments' 34 | 35 | response["time"]["store-review"]["end_time"] = time() 36 | print(response) 37 | except Exception as e: 38 | print(e) 39 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 40 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 41 | else: 42 | print("success") 43 | 44 | 45 | return response 46 | 47 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/upload_movie_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | import ray 9 | 10 | 11 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 12 | class UploadMovieReviewService(object): 13 | # def __init__(self): 14 | def UploadMovieReview(self, body1: Dict): 15 | print(123) 16 | try: 17 | start = time() 18 | 19 | myclient = pymongo.MongoClient("mongodb://movie-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 20 | mydb = myclient["movie-review"] 21 | mycol = mydb["movie-review"] 22 | 23 | event = body1 24 | print(event) 25 | response = {"time": {"upload-movie-review": {"start_time": start}}} 26 | 27 | try: 28 | movie_id = event["body"]["movie_id"] 29 | review_id = event["body"]["review_id"] 30 | timestamp = event["body"]["timestamp"] 31 | 32 | myquery = {"movie_id": movie_id} 33 | mydoc = mycol.find(myquery) 34 | if mycol.count_documents(myquery) > 0: 35 | reviews = json.loads(mydoc.next()["reviews"]) 36 | reviews.append((review_id, timestamp)) 37 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 38 | mycol.update_one(myquery, reviews_update) 39 | else: 40 | body = {"movie_id": movie_id, "reviews": json.dumps([(review_id, timestamp)])} 41 | mycol.insert_one(body) 42 | except Exception as e: 43 | response['body'] = 'Incomplete arguments' 44 | 45 | response["time"]["upload-movie-review"]["end_time"] = time() 46 | print(response) 47 | except Exception as e: 48 | print(e) 49 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 50 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 51 | else: 52 | print("success") 53 | 54 | 55 | return response 56 | 57 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/upload_user_id.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import ray 4 | from time import time 5 | import pymongo 6 | from pymemcache.client.base import Client 7 | import json 8 | 9 | mongo_url = "mongodb://user-mongodb.movie-reviewing.svc.cluster.local:27017/" 10 | memcache_url = "user-memcached.movie-reviewing.svc.cluster.local:11211" 11 | 12 | mc = Client(memcache_url) 13 | 14 | arguments = ["title", "text", "username", "password", "rating"] 15 | 16 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 17 | class UploadUserIdService(object): 18 | # def __init__(self): 19 | def UploadUserId(self, body1: Dict): 20 | print(123) 21 | try: 22 | start = time() 23 | 24 | myclient = pymongo.MongoClient(mongo_url) 25 | mydb = myclient['user'] 26 | mycol = mydb["user"] 27 | print(body1) 28 | event = body1 29 | body = '' 30 | username = '' 31 | response = {"time": {"upload-user-id": {"start_time": start}}} 32 | try: 33 | username = event["body"]["username"] 34 | except Exception as e: 35 | body = 'Incomplete arguments' 36 | 37 | if username: 38 | user_id = -1 39 | mmc_user_id = mc.get(username + ":user_id") 40 | if mmc_user_id != None: 41 | user_id = mmc_user_id 42 | body = {"user_id": user_id} 43 | else: 44 | myquery = {"username": username} 45 | mydoc = mycol.find(myquery) 46 | print("query num:",mycol.count_documents(myquery)) 47 | if mycol.count_documents(myquery) > 0: 48 | it = mydoc.next() 49 | user_id = it["user_id"] 50 | body = {"user_id": user_id} 51 | mc.set(username + ":user_id", user_id) 52 | else: 53 | body = 'No user ' + username 54 | 55 | response["body"] = body 56 | 57 | response["time"]["upload-user-id"]["end_time"] = time() 58 | print(response) 59 | 60 | except Exception as e: 61 | print(e) 62 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 63 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 64 | else: 65 | print("success") 66 | 67 | 68 | return ray.put(response) 69 | -------------------------------------------------------------------------------- /media_service/ray_stateful/ms/service/upload_user_review.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from time import time 6 | import pymongo 7 | import json 8 | import ray 9 | 10 | 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class UploadUserReviewService(object): 14 | # def __init__(self): 15 | def UploadUserReview(self, body1: Dict): 16 | print(123) 17 | try: 18 | start = time() 19 | 20 | myclient = pymongo.MongoClient("mongodb://user-review-mongodb.movie-reviewing.svc.cluster.local:27017/") 21 | mydb = myclient["user-review"] 22 | mycol = mydb["user-review"] 23 | 24 | event = body1 25 | print(event) 26 | response = {"time": {"upload-user-review": {"start_time": start}}} 27 | 28 | try: 29 | user_id = event["body"]["user_id"] 30 | review_id = event["body"]["review_id"] 31 | timestamp = event["body"]["timestamp"] 32 | 33 | myquery = {"user_id": user_id} 34 | mydoc = mycol.find(myquery) 35 | if mycol.count_documents(myquery) > 0: 36 | reviews = json.loads(mydoc.next()["reviews"]) 37 | reviews.append((review_id, timestamp)) 38 | reviews_update = {"$set": {"reviews": json.dumps(reviews)}} 39 | mycol.update_one(myquery, reviews_update) 40 | else: 41 | body = {"user_id": user_id, "reviews": json.dumps([(review_id, timestamp)])} 42 | mycol.insert_one(body) 43 | except Exception as e: 44 | response['body'] = 'Incomplete arguments' 45 | 46 | response["time"]["upload-user-review"]["end_time"] = time() 47 | except Exception as e: 48 | print(e) 49 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 50 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 51 | else: 52 | print("success") 53 | 54 | 55 | return response 56 | 57 | -------------------------------------------------------------------------------- /solver/openfaas/.gitignore: -------------------------------------------------------------------------------- 1 | template 2 | build 3 | .secrets 4 | -------------------------------------------------------------------------------- /solver/openfaas/arithmetic-arranger.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | arithmetic-arranger: 7 | lang: python3-debian 8 | handler: ./arithmetic-arranger 9 | image: 13051172893/arithmetic-arranger:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/arithmetic-arranger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/arithmetic-arranger/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/arithmetic-arranger/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/arithmetic-arranger/requirements.txt -------------------------------------------------------------------------------- /solver/openfaas/curve-fitting.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | curve-fitting: 7 | lang: python3-debian 8 | handler: ./curve-fitting 9 | image: 13051172893/curve-fitting:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/curve-fitting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/curve-fitting/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/curve-fitting/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | from scipy.optimize import curve_fit 3 | 4 | 5 | def handle(req): 6 | """handle a request to the function 7 | Args: 8 | req (str): request body 9 | """ 10 | event = json.loads(req) 11 | X = event['X'] 12 | y = event['y'] 13 | 14 | def func(x, a, b): 15 | y = a * x + b 16 | return y 17 | 18 | try: 19 | popt, pcov = curve_fit(func, X, y) 20 | except Exception as e: 21 | print(e) 22 | return e 23 | else: 24 | return popt, pcov 25 | -------------------------------------------------------------------------------- /solver/openfaas/curve-fitting/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy -------------------------------------------------------------------------------- /solver/openfaas/eigen.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | eigen: 7 | lang: python3-debian 8 | handler: ./eigen 9 | image: 13051172893/eigen:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/eigen/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/eigen/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/eigen/handler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import json 3 | 4 | 5 | def handle(req): 6 | event = json.loads(req) 7 | matrix = event["matrix"] 8 | eigenvalue, featurevector = np.linalg.eig(matrix) 9 | return { 10 | "eigenvalue": eigenvalue, 11 | "featurevector": featurevector 12 | } 13 | -------------------------------------------------------------------------------- /solver/openfaas/eigen/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy -------------------------------------------------------------------------------- /solver/openfaas/linear-programming.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | linear-programming: 7 | lang: python3-debian 8 | handler: ./linear-programming 9 | image: 13051172893/linear-programming:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/linear-programming/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/linear-programming/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/linear-programming/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | from scipy.optimize import linprog 4 | 5 | 6 | def PrintRes(res): 7 | return { 8 | "OptimalValue": res.fun, 9 | "OptimalSolution": res.x 10 | } 11 | 12 | 13 | def handle(req): 14 | """handle a request to the function 15 | Args: 16 | req (str): request body 17 | """ 18 | event = json.loads(req) 19 | MinOrMax = event['MinOrMax'] 20 | target = event['target'] 21 | A = event['A'] 22 | b = event['b'] 23 | bounds = event['bounds'] 24 | print("线性规划求解器:") 25 | if MinOrMax == 'min': 26 | pass 27 | elif MinOrMax == 'max': 28 | target = np.array(target) * (-1) 29 | # minimize 30 | res = linprog(target, A, b, bounds=bounds) 31 | return PrintRes(res) 32 | -------------------------------------------------------------------------------- /solver/openfaas/linear-programming/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scipy -------------------------------------------------------------------------------- /solver/openfaas/prob-calculator.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | prob-calculator: 7 | lang: python3-debian 8 | handler: ./prob-calculator 9 | image: 13051172893/prob-calculator:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/prob-calculator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/prob-calculator/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/prob-calculator/handler.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import random 3 | import json 4 | 5 | 6 | # Consider using the modules imported above. 7 | 8 | class Hat: 9 | 10 | def __init__(self, **kwargs): 11 | self.contents = [] 12 | for key, value in kwargs.items(): 13 | for _ in range(value): 14 | self.contents.append(key) 15 | 16 | def draw(self, number): 17 | if number > len(self.contents): 18 | return self.contents 19 | balls = [] 20 | for _ in range(number): 21 | choice = random.randrange(len(self.contents)) 22 | balls.append(self.contents.pop(choice)) 23 | return balls 24 | 25 | 26 | def experiment(hat, expected_balls, num_balls_drawn, num_experiments): 27 | expected_no_of_balls = [] 28 | for key in expected_balls: 29 | expected_no_of_balls.append(expected_balls[key]) 30 | successes = 0 31 | 32 | for _ in range(num_experiments): 33 | new_hat = copy.deepcopy(hat) 34 | balls = new_hat.draw(num_balls_drawn) 35 | 36 | no_of_balls = [] 37 | for key in expected_balls: 38 | no_of_balls.append(balls.count(key)) 39 | 40 | if no_of_balls >= expected_no_of_balls: 41 | successes += 1 42 | 43 | return successes / num_experiments 44 | 45 | 46 | def handle(req): 47 | event = json.loads(req) 48 | hat = Hat(**event["hat"]) 49 | probability = experiment( 50 | hat=hat, 51 | expected_balls=event["expected_balls"], 52 | num_balls_drawn=event["num_balls_drawn"], 53 | num_experiments=event["num_experiments"] 54 | ) 55 | return probability 56 | -------------------------------------------------------------------------------- /solver/openfaas/prob-calculator/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/prob-calculator/requirements.txt -------------------------------------------------------------------------------- /solver/openfaas/readme.md: -------------------------------------------------------------------------------- 1 | - arithmetic-arranger：算式格式化 2 | - Example：curl http://127.0.0.1:31128/function/arithmetic-arranger.openfaas-fn-pyz --data-binary '{"problems":["32 + 698", "3801 - 2", "45 + 43", "123 + 49"]}' 3 | 4 | - curve-fitting：线性拟合 5 | - curl http://127.0.0.1:31128/function/curve-fitting.openfaas-fn-pyz --data-binary '{"X":[150, 200, 250, 350, 300, 400, 600],"y":[6450, 7450, 8450, 9450, 11450, 15450, 18450]}' 6 | 7 | - eigen：求解矩阵特征值和特征向量 8 | - curl http://127.0.0.1:31128/function/eigen.openfaas-fn-pyz --data-binary '{"matrix":[[-1, 1, 0],[-4, 3, 0],[1, 0, 2]]}' 9 | 10 | - linear-programming：线性规划求解 11 | - Example：curl http://127.0.0.1:31128/function/linear-programming.openfaas-fn-pyz --data-binary '{"MinOrMax":"min","target":[-1,4],"A":[[-3,1],[1,2]],"b":[6,4],"bounds":[[null,null],[-3,null]]}' 12 | 13 | - prob-calculator：计算古典概型概率 14 | - curl http://127.0.0.1:31128/function/prob-calculator.openfaas-fn-pyz --data-binary '{"hat":{"blue":4,"red":2,"green":6},"expected_balls":{"blue":2,"red":1},"num_balls_drawn":4,"num_experiments":3000}' 15 | 16 | - time-calculator：日期计算 17 | - curl http://127.0.0.1:31128/function/time-calculator.openfaas-fn-pyz --data-binary '{"start":"11:06 PM","duration":"2:02"}' -------------------------------------------------------------------------------- /solver/openfaas/time-calculator.yml: -------------------------------------------------------------------------------- 1 | version: 1.0 2 | provider: 3 | name: openfaas 4 | gateway: http://127.0.0.1:31128 5 | functions: 6 | time-calculator: 7 | lang: python3-debian 8 | handler: ./time-calculator 9 | image: 13051172893/time-calculator:latest 10 | 11 | -------------------------------------------------------------------------------- /solver/openfaas/time-calculator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/time-calculator/__init__.py -------------------------------------------------------------------------------- /solver/openfaas/time-calculator/handler.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | def add_time(start, duration, starting_day=""): 5 | # Separte the start into hours and minutes 6 | pieces = start.split() 7 | time = pieces[0].split(":") 8 | end = pieces[1] 9 | 10 | # Calculate 24-hour clock format 11 | if end == "PM": 12 | hour = int(time[0]) + 12 13 | time[0] = str(hour) 14 | 15 | # Separate the duration into hours and minutes 16 | dur_time = duration.split(":") 17 | 18 | # Add hours and minutes 19 | new_hour = int(time[0]) + int(dur_time[0]) 20 | new_minutes = int(time[1]) + int(dur_time[1]) 21 | 22 | if new_minutes >= 60: 23 | hours_add = new_minutes // 60 24 | new_minutes -= hours_add * 60 25 | new_hour += hours_add 26 | 27 | days_add = 0 28 | if new_hour > 24: 29 | days_add = new_hour // 24 30 | new_hour -= days_add * 24 31 | 32 | # Find AM and PM 33 | # Return to 12-hour clock format 34 | if new_hour > 0 and new_hour < 12: 35 | end = "AM" 36 | elif new_hour == 12: 37 | end = "PM" 38 | elif new_hour > 12: 39 | end = "PM" 40 | new_hour -= 12 41 | else: # new_hour == 0 42 | end = "AM" 43 | new_hour += 12 44 | 45 | if days_add > 0: 46 | if days_add == 1: 47 | days_later = " (next day)" 48 | else: 49 | days_later = " (" + str(days_add) + " days later)" 50 | else: 51 | days_later = "" 52 | 53 | week_days = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday") 54 | 55 | if starting_day: 56 | weeks = days_add // 7 57 | i = week_days.index(starting_day.lower().capitalize()) + (days_add - 7 * weeks) 58 | if i > 6: 59 | i -= 7 60 | day = ", " + week_days[i] 61 | else: 62 | day = "" 63 | 64 | new_time = str(new_hour) + ":" + \ 65 | (str(new_minutes) if new_minutes > 9 else ("0" + str(new_minutes))) + \ 66 | " " + end + day + days_later 67 | 68 | return new_time 69 | 70 | 71 | def handle(req): 72 | event = json.loads(req) 73 | return add_time(event["start"], event["duration"]) 74 | -------------------------------------------------------------------------------- /solver/openfaas/time-calculator/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/openfaas/time-calculator/requirements.txt -------------------------------------------------------------------------------- /solver/ray_stateful/so/model/post.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | from typing import List, Set, Union 4 | 5 | 6 | 7 | 8 | class PostItem(BaseModel): 9 | portfolioType: str 10 | portfolio: str 11 | 12 | 13 | class PostStr(BaseModel): 14 | input_str: str 15 | 16 | 17 | class PostBodyMap(BaseModel): 18 | input_name: str 19 | input_part: int 20 | reduce_num: int 21 | 22 | 23 | class PostBodyReduce(BaseModel): 24 | input_name: str 25 | input_num: int 26 | reduce_part: int 27 | -------------------------------------------------------------------------------- /solver/ray_stateful/so/requirements/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ServerlessOS/Yet-Another-Serverless-Benchmark/c7c094cf521ef807671b93440643ab81483f41e0/solver/ray_stateful/so/requirements/requirements.txt -------------------------------------------------------------------------------- /solver/ray_stateful/so/service/curve_fitting.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | 5 | from scipy.optimize import curve_fit 6 | 7 | 8 | 9 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 10 | class CurveFittingService(object): 11 | # def __init__(self): 12 | def CurveFitting(self, body: Dict): 13 | print(123) 14 | try: 15 | 16 | event = body 17 | X = event['X'] 18 | y = event['y'] 19 | 20 | def func(x, a, b): 21 | y = a * x + b 22 | return y 23 | popt, pcov = curve_fit(func, X, y) 24 | 25 | except Exception as e: 26 | print(e) 27 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 28 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 29 | else: 30 | print("success") 31 | 32 | 33 | return {"popt":popt, "pcov":pcov} 34 | 35 | -------------------------------------------------------------------------------- /solver/ray_stateful/so/service/eigen.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | import numpy as np 4 | import json 5 | 6 | 7 | 8 | 9 | 10 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 11 | class EigenService(object): 12 | # def __init__(self): 13 | def Eigen(self, body: Dict): 14 | print(123) 15 | try: 16 | 17 | event = body 18 | matrix = event["matrix"] 19 | eigenvalue, featurevector = np.linalg.eig(matrix) 20 | print("特征值：", eigenvalue) 21 | print("特征向量：", featurevector) 22 | 23 | except Exception as e: 24 | print(e) 25 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 26 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 27 | else: 28 | print("success") 29 | 30 | return { 31 | "eigenvalue": eigenvalue, 32 | "featurevector": featurevector 33 | } 34 | 35 | -------------------------------------------------------------------------------- /solver/ray_stateful/so/service/linear_programming.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | import json 5 | import numpy as np 6 | from scipy.optimize import linprog 7 | 8 | 9 | 10 | 11 | 12 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 13 | class LinearProgrammingService(object): 14 | # def __init__(self): 15 | def LinearProgramming(self, body: Dict): 16 | print(123) 17 | try: 18 | 19 | event = body 20 | MinOrMax = event['MinOrMax'] 21 | target = event['target'] 22 | A = event['A'] 23 | b = event['b'] 24 | bounds = event['bounds'] 25 | print("线性规划求解器:") 26 | if MinOrMax == 'min': 27 | pass 28 | elif MinOrMax == 'max': 29 | target = np.array(target) * (-1) 30 | # minimize 31 | res = linprog(target, A, b, bounds=bounds) 32 | 33 | except Exception as e: 34 | print(e) 35 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 36 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 37 | else: 38 | print("success") 39 | 40 | return { 41 | "OptimalValue": res.fun, 42 | "OptimalSolution": res.x 43 | } 44 | 45 | -------------------------------------------------------------------------------- /solver/ray_stateful/so/service/prob_calculator.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | import copy 5 | import random 6 | import json 7 | 8 | 9 | class Hat: 10 | 11 | def __init__(self, **kwargs): 12 | self.contents = [] 13 | for key, value in kwargs.items(): 14 | for _ in range(value): 15 | self.contents.append(key) 16 | 17 | def draw(self, number): 18 | if number > len(self.contents): 19 | return self.contents 20 | balls = [] 21 | for _ in range(number): 22 | choice = random.randrange(len(self.contents)) 23 | balls.append(self.contents.pop(choice)) 24 | return balls 25 | 26 | 27 | def experiment(hat, expected_balls, num_balls_drawn, num_experiments): 28 | expected_no_of_balls = [] 29 | for key in expected_balls: 30 | expected_no_of_balls.append(expected_balls[key]) 31 | successes = 0 32 | 33 | for _ in range(num_experiments): 34 | new_hat = copy.deepcopy(hat) 35 | balls = new_hat.draw(num_balls_drawn) 36 | 37 | no_of_balls = [] 38 | for key in expected_balls: 39 | no_of_balls.append(balls.count(key)) 40 | 41 | if no_of_balls >= expected_no_of_balls: 42 | successes += 1 43 | 44 | return successes / num_experiments 45 | 46 | 47 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 48 | class ProbCalculatorService(object): 49 | # def __init__(self): 50 | def ProbCalculator(self, body: Dict): 51 | print(123) 52 | try: 53 | 54 | event = body 55 | hat = Hat(**event["hat"]) 56 | probability = experiment( 57 | hat=hat, 58 | expected_balls=event["expected_balls"], 59 | num_balls_drawn=event["num_balls_drawn"], 60 | num_experiments=event["num_experiments"] 61 | ) 62 | 63 | 64 | except Exception as e: 65 | print(e) 66 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 67 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 68 | else: 69 | print("success") 70 | 71 | return probability 72 | 73 | -------------------------------------------------------------------------------- /solver/ray_stateful/so/service/time_calculator.py: -------------------------------------------------------------------------------- 1 | from ray import serve 2 | from typing import List, Dict 3 | 4 | import json 5 | 6 | 7 | 8 | def add_time(start, duration, starting_day=""): 9 | # Separte the start into hours and minutes 10 | pieces = start.split() 11 | time = pieces[0].split(":") 12 | end = pieces[1] 13 | 14 | # Calculate 24-hour clock format 15 | if end == "PM": 16 | hour = int(time[0]) + 12 17 | time[0] = str(hour) 18 | 19 | # Separate the duration into hours and minutes 20 | dur_time = duration.split(":") 21 | 22 | # Add hours and minutes 23 | new_hour = int(time[0]) + int(dur_time[0]) 24 | new_minutes = int(time[1]) + int(dur_time[1]) 25 | 26 | if new_minutes >= 60: 27 | hours_add = new_minutes // 60 28 | new_minutes -= hours_add * 60 29 | new_hour += hours_add 30 | 31 | days_add = 0 32 | if new_hour > 24: 33 | days_add = new_hour // 24 34 | new_hour -= days_add * 24 35 | 36 | # Find AM and PM 37 | # Return to 12-hour clock format 38 | if new_hour > 0 and new_hour < 12: 39 | end = "AM" 40 | elif new_hour == 12: 41 | end = "PM" 42 | elif new_hour > 12: 43 | end = "PM" 44 | new_hour -= 12 45 | else: # new_hour == 0 46 | end = "AM" 47 | new_hour += 12 48 | 49 | if days_add > 0: 50 | if days_add == 1: 51 | days_later = " (next day)" 52 | else: 53 | days_later = " (" + str(days_add) + " days later)" 54 | else: 55 | days_later = "" 56 | 57 | week_days = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday") 58 | 59 | if starting_day: 60 | weeks = days_add // 7 61 | i = week_days.index(starting_day.lower().capitalize()) + (days_add - 7 * weeks) 62 | if i > 6: 63 | i -= 7 64 | day = ", " + week_days[i] 65 | else: 66 | day = "" 67 | 68 | new_time = str(new_hour) + ":" + \ 69 | (str(new_minutes) if new_minutes > 9 else ("0" + str(new_minutes))) + \ 70 | " " + end + day + days_later 71 | 72 | return new_time 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | @serve.deployment(num_replicas=1, ray_actor_options={"num_cpus": 1, "num_gpus": 0}) 82 | class TimeCalculatorService(object): 83 | # def __init__(self): 84 | def TimeCalculator(self, body: Dict): 85 | print(123) 86 | try: 87 | event = body 88 | except Exception as e: 89 | print(e) 90 | print(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件 91 | print(e.__traceback__.tb_lineno) # 发生异常所在的行数 92 | else: 93 | print("success") 94 | 95 | return add_time(event["start"], event["duration"]) 96 | 97 | -------------------------------------------------------------------------------- /solver/ray_stateful/solver.yaml: -------------------------------------------------------------------------------- 1 | import_path: so.main:ingress 2 | 3 | runtime_env: 4 | pip: 5 | - "numpy" 6 | - "scipy" 7 | 8 | host: 0.0.0.0 9 | 10 | port: 8000 11 | 12 | deployments: 13 | 14 | - name: ArithmeticArrangerService 15 | num_replicas: 1 16 | ray_actor_options: 17 | num_cpus: 1.0 18 | num_gpus: 0.0 19 | 20 | - name: CurveFittingService 21 | num_replicas: 1 22 | ray_actor_options: 23 | num_cpus: 1.0 24 | num_gpus: 0.0 25 | 26 | - name: EigenService 27 | num_replicas: 1 28 | ray_actor_options: 29 | num_cpus: 1.0 30 | num_gpus: 0.0 31 | 32 | - name: LinearProgrammingService 33 | num_replicas: 1 34 | ray_actor_options: 35 | num_cpus: 1.0 36 | num_gpus: 0.0 37 | 38 | - name: ProbCalculatorService 39 | num_replicas: 1 40 | ray_actor_options: 41 | num_cpus: 1.0 42 | num_gpus: 0.0 43 | 44 | - name: TimeCalculatorService 45 | num_replicas: 1 46 | ray_actor_options: 47 | num_cpus: 1.0 48 | num_gpus: 0.0 49 | 50 | 51 | 52 | 53 | - name: Solver --------------------------------------------------------------------------------