├── requirements.txt ├── Makefile ├── documents ├── titaniadb-sentinel.jpg ├── titaniadb-sentinel.png └── titaniadb_architecture.PNG ├── image └── titaniadb_sentinel_fig.png ├── test ├── images │ └── titaniadb-sentinel-test.png ├── documents │ └── titaniadb-sentinel-test.png ├── init-mysql.sh ├── db │ └── mysql_init │ │ └── make_databases.sql ├── README.md ├── docker-compose.yml └── src │ └── test_etcd.py ├── shell ├── exec.sh └── run.sh ├── src ├── examples │ ├── mysql.py │ ├── data.py │ └── etcd.py ├── etcd.py ├── main.py └── mysql.py ├── Dockerfile ├── k8s ├── production │ └── deployment.yml └── development │ └── deployment.yml ├── docker-build.sh ├── LICENSE └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | etcd3 2 | mysqlclient==1.4.6 -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | docker-build: 2 | bash ./docker-build.sh 3 | 4 | docker-push: 5 | bash ./docker-build.sh push -------------------------------------------------------------------------------- /documents/titaniadb-sentinel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/documents/titaniadb-sentinel.jpg -------------------------------------------------------------------------------- /documents/titaniadb-sentinel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/documents/titaniadb-sentinel.png -------------------------------------------------------------------------------- /image/titaniadb_sentinel_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/image/titaniadb_sentinel_fig.png -------------------------------------------------------------------------------- /documents/titaniadb_architecture.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/documents/titaniadb_architecture.PNG -------------------------------------------------------------------------------- /test/images/titaniadb-sentinel-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/test/images/titaniadb-sentinel-test.png -------------------------------------------------------------------------------- /test/documents/titaniadb-sentinel-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/latonaio/titaniadb-sentinel/HEAD/test/documents/titaniadb-sentinel-test.png -------------------------------------------------------------------------------- /shell/exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | POD_ID=$(kubectl get po | awk '{print $1}' | grep -v NAME | grep titaniadb-sentinel) 4 | kubectl exec -it ${POD_ID} -- bash -------------------------------------------------------------------------------- /shell/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | POD_ID=$(kubectl get po | awk '{print $1}' | grep -v NAME | grep titaniadb-sentinel) 4 | kubectl exec -it ${POD_ID} -- python3 -u main.py -------------------------------------------------------------------------------- /test/init-mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker-compose exec db bash -c "chmod 0775 docker-entrypoint-initdb.d/init-database.sh" 3 | docker-compose exec db bash -c "./docker-entrypoint-initdb.d/init-database.sh" 4 | -------------------------------------------------------------------------------- /src/examples/mysql.py: -------------------------------------------------------------------------------- 1 | from src import mysql 2 | from src.examples import data 3 | 4 | 5 | def main(): 6 | with mysql.ProjectTable() as client: 7 | client.create_database() 8 | client.create_table() 9 | client.upsert(data.project1) 10 | return 11 | 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM l4t:latest 2 | 3 | # Definition of a Device & Service 4 | ENV POSITION=Runtime \ 5 | SERVICE=titaniadb-sentinel \ 6 | AION_HOME=/var/lib/aion \ 7 | MY_MYSQL_HOST=mysql \ 8 | MY_MYSQL_USER=MYSQL_USER_XXX \ 9 | MY_ETCD_HOST=titaniadb 10 | 11 | RUN apt-get update && apt-get install -y libmysqlclient-dev 12 | 13 | # Setup Directoties 14 | RUN mkdir -p \ 15 | $POSITION/$SERVICE 16 | WORKDIR ${AION_HOME}/$POSITION/$SERVICE/ 17 | 18 | ADD requirements.txt . 19 | 20 | RUN pip3 install -r requirements.txt 21 | 22 | ADD src/ . 23 | 24 | CMD ["python3", "-u", "main.py"] 25 | -------------------------------------------------------------------------------- /k8s/production/deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: titaniadb-sentinel 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | role: titaniadb-sentinel 10 | template: 11 | metadata: 12 | labels: 13 | role: titaniadb-sentinel 14 | spec: 15 | shareProcessNamespace: true 16 | containers: 17 | - name: titaniadb-sentinel 18 | image: latonaio/titaniadb-sentinel:latest 19 | imagePullPolicy: IfNotPresent 20 | env: 21 | - name: MY_MYSQL_PASSWORD 22 | value: MYSQL_PASSWORD_XXX 23 | -------------------------------------------------------------------------------- /test/db/mysql_init/make_databases.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE Device; 2 | CREATE DATABASE Pod; 3 | use Device; 4 | 5 | CREATE TABLE device ( 6 | deviceName VARCHAR(20), 7 | projectSymbolFk VARCHAR(20), 8 | deviceIp VARCHAR(20), 9 | connectionStatus INT NOT NULL PRIMARY KEY, 10 | os VARCHAR(20)); 11 | 12 | use Pod; 13 | 14 | CREATE TABLE pod( 15 | podName VARCHAR(20)NOT NULL PRIMARY KEY, 16 | deviceNameFk VARCHAR(20), 17 | imageName VARCHAR(20), 18 | currentVersion VARCHAR(20) , 19 | latestVersion VARCHAR(20), 20 | deployedAt TIMESTAMP, 21 | status VARCHAR(20)); -------------------------------------------------------------------------------- /docker-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | PUSH=$1 4 | DATE="$(date "+%Y%m%d%H%M")" 5 | REPOSITORY_PREFIX="latonaio" 6 | SERVICE_NAME="titaniadb-sentinel" 7 | 8 | DOCKER_BUILDKIT=1 docker build --progress=plain -t ${SERVICE_NAME}:"${DATE}" . 9 | 10 | # tagging 11 | docker tag ${SERVICE_NAME}:"${DATE}" ${SERVICE_NAME}:latest 12 | docker tag ${SERVICE_NAME}:"${DATE}" ${REPOSITORY_PREFIX}/${SERVICE_NAME}:"${DATE}" 13 | docker tag ${REPOSITORY_PREFIX}/${SERVICE_NAME}:"${DATE}" ${REPOSITORY_PREFIX}/${SERVICE_NAME}:latest 14 | 15 | if [[ $PUSH == "push" ]]; then 16 | docker push ${REPOSITORY_PREFIX}/${SERVICE_NAME}:"${DATE}" 17 | docker push ${REPOSITORY_PREFIX}/${SERVICE_NAME}:latest 18 | fi -------------------------------------------------------------------------------- /k8s/development/deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: titaniadb-sentinel 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | role: titaniadb-sentinel 10 | template: 11 | metadata: 12 | labels: 13 | role: titaniadb-sentinel 14 | spec: 15 | containers: 16 | - name: titaniadb-sentinel 17 | command: ["/bin/sh", "-c", "while :; do sleep 10000; done"] 18 | image: latonaio/titaniadb-sentinel:latest 19 | imagePullPolicy: IfNotPresent 20 | env: 21 | - name: MY_MYSQL_PASSWORD 22 | value: MYSQL_PASSWORD_XXX 23 | volumeMounts: 24 | - name: src 25 | mountPath: /var/lib/aion/Runtime/titaniadb-sentinel/ 26 | volumes: 27 | - name: src 28 | hostPath: 29 | path: /home/latona/vega/Runtime/titaniadb-sentinel/src 30 | -------------------------------------------------------------------------------- /src/examples/data.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | 4 | project1 = { 5 | 'projectSymbol': '1234567890abcdef', 6 | 'projectName': 'exampleProject', 7 | 'projectID': 'exampleProjectID', 8 | 'timestamp': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 9 | } 10 | 11 | device1 = { 12 | 'deviceName': 'titania', 13 | 'projectSymbolFk': '1234567890abcdef', 14 | 'deviceIp': '192.168.128.161', 15 | 'connectionStatus': 1, 16 | 'os': 'ubuntu:18.04' 17 | } 18 | 19 | pod1 = { 20 | 'podName': 'podName', 21 | 'deviceNameFk': 'titania', 22 | 'imageName': 'nginx', 23 | 'currentVersion': 'currentVersion', 24 | 'latestVersion': 'latestVersion', 25 | 'deployedAt': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 26 | 'status': 'Running' 27 | } 28 | 29 | pod2 = { 30 | 'podName': 'podName2', 31 | 'deviceNameFk': 'titania', 32 | 'currentVersion': 'currentVersion', 33 | 'latestVersion': 'latestVersion', 34 | 'deployedAt': '', 35 | 'status': 'Running' 36 | } 37 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | ## titaniadb-sentinel-test 2 | titaniadb-sentinel-testは、titaniadb-sentinelのテスト用mockサーバーです。 3 | 4 | 5 | ## 概要 6 | titaniadb-sentinel-testは、etcd、MySQL、ubuntuの三つのコンテナによって構成されています。titaniadb-sentinel-testを使用することで、titaniadb-sentinelの動作とOutputの確認を行うことができます。 7 | 8 | ## 起動方法 9 | 以下のコマンドによりtitaniadb-sentinel-testを起動してください。 10 | 11 | ``` 12 | $ cd test 13 | $ docker-compose up 14 | ``` 15 | 16 | ## Input 17 | JSON形式のデータ。etcdに送られるデータの内容はtest_etcd.pyに記載されています。 18 | 19 | ## Output 20 | MySQLへのデータ入力。MySQL内に保存されたデータは以下のコマンドによって確認することができます。 21 | 22 | ``` 23 | $ cd titaniadb-sentinel 24 | $ docker exec -it [コンテナ名] bash 25 | $ docker mysql -u root -p 26 | $ use [データベース名] 27 | $ select * from [テーブル名] 28 | ``` 29 | 30 | ## システム構造 31 | ![System Configuration](documents/titaniadb-sentinel-test.png) 32 | 33 | ## 使用したdocker image 34 | ### bitnami/etcd 35 | 　-https://hub.docker.com/r/bitnami/etcd/dockerfile 36 | ### ubuntu 37 | -https://hub.docker.com/_/ubuntu 38 | ### mysql 39 | -https://hub.docker.com/_/mysql 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Latona, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | etcd: 5 | image: bitnami/etcd:3.4.3 6 | environment: 7 | - ALLOW_NONE_AUTHENTICATION=yes 8 | - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 9 | ports: 10 | - 2379:2379 11 | - 2380:2380 12 | networks: 13 | - etcdtest 14 | 15 | db: 16 | image: mysql:5.7 17 | environment: 18 | MYSQL_ROOT_PASSWORD: "root" 19 | MYSQL_DATABASE: "mock_test_database" 20 | MYSQL_USER: "dbtest" 21 | MYSQL_PASSWORD: "pass_for_dbtest" 22 | TZ: 'Asia/Tokyo' 23 | command: mysqld --innodb-use-native-aio=0 24 | volumes: 25 | - ./db/mysql_init:/docker-entrypoint-initdb.d 26 | - db-data:/var/lib/mysql 27 | ports: 28 | - 3306:3306 29 | 30 | docker_etcd_test: 31 | image: ubuntu 32 | dns: 8.8.8.8 33 | environment: 34 | TZ: 'Asia/Tokyo' 35 | volumes: 36 | - ./src:/home/src 37 | working_dir: /home/src 38 | command: bash -c "apt-get update && apt-get install -y python3 python3-pip && apt-get update && apt-get update -y && pip3 install etcd3 && python3 -u test_etcd.py" 39 | networks: 40 | - etcdtest 41 | 42 | volumes: 43 | db-data: 44 | driver: local 45 | 46 | networks: 47 | etcdtest: 48 | name: etcdtest 49 | driver: bridge 50 | -------------------------------------------------------------------------------- /test/src/test_etcd.py: -------------------------------------------------------------------------------- 1 | import etcd3 2 | import os 3 | import datetime 4 | import time 5 | import json 6 | 7 | pr1 = { 8 | 'projectSymbol': '1234567890abcdef', 9 | 'projectName': 'exampleProject', 10 | 'projectID': 'exampleProjectID', 11 | 'timestamp': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 12 | } 13 | 14 | d1 = { 15 | 'deviceName': 'titania', #'titania' 16 | 'projectSymbolFk': '1234567890abcdef', 17 | 'deviceIp':'192.168.128.161', # 18 | 'connectionStatus': repr(1), #'1' 19 | 'os': 'ubuntu:18.04' 20 | } 21 | 22 | p1 = { 23 | 'podName': 'podName', 24 | 'deviceNameFk': 'titania', 25 | 'imageName': 'nginx', 26 | 'currentVersion': 'currentVersion', #'currentVersion' 27 | 'latestVersion': 'latestVersion', 28 | 'deployedAt': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 29 | 'status': 'Running' 30 | } 31 | 32 | p2 = { 33 | 'podName': 'podName2', 34 | 'deviceNameFk': 'titania', 35 | 'currentVersion': 'currentVersion', 36 | 'latestVersion': 'latestVersion', 37 | 'deployedAt': '', 38 | 'status': 'Running' 39 | } 40 | 41 | #convert to json 42 | pr1_j = json.dumps(pr1) 43 | d1_j = json.dumps(d1) 44 | p1_j = json.dumps(p1) 45 | p2_j = json.dumps(p2) 46 | 47 | project1 = ["/Project/1/1", pr1_j] 48 | device1 = ["/Device/1/1", d1_j] 49 | pod1 = ["/Pod/1/1", p1_j] 50 | pod2 = ["/Pod/2/1", p2_j] 51 | 52 | 53 | client = etcd3.client(host = 'etcd', port = 2379) 54 | KV_list = [project1, device1, pod1, pod2] 55 | 56 | for KVs in KV_list : 57 | Key = KVs[0] 58 | Value = KVs[1] 59 | client.put(Key, Value) 60 | time.sleep(5) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |

3 | 4 | *** 5 | 6 | # titaniadb-sentinel 7 | titaniadb-sentinelは、エッジ上で、titaniadb(etcd) を対象として、データベースの監視、制御を行うマイクロサービスです。 8 | ![titaniadb-sentinel](documents/titaniadb_architecture.PNG) 9 | 10 | ## Description 11 | titaniadb-sentinel は、etcdをラップしたエッジ上のtitaniadbと連携します。 12 | このマイクロサービスは、titaniadb-sentinel が収集したデータが蓄積されたtitaniadbにおいて、データの置換、挿入が正しく行われるよう監視、制御することができます。 13 | 14 | titaniadbは、エッジ上の IP、MACアドレス、kubernetesノードの状態、ポッドの状態など、エッジ間の安定したデータインタフェースや処理に重要なIoTメタデータをデータベースに蓄積しますが、titaniadb-sentinelは、エッジ上で、これらのデータの置換、挿入が正しく行われるよう巡回、制御します。 15 | 16 | 17 | ## 依存関係 18 | 19 | - [distributed-service-discovery](https://github.com/latonaio/distributed-service-discovery) 20 | - [gossip-propagation-d](https://github.com/latonaio/gossip-propagation-d) 21 | 22 | 23 | ## etcd 24 | etcdは、オープンソースで分散型のキーバリューストアです。etcd はkubernetes のプライマリーデータストアとして採用されており、kubernetesクラスタの情報を保存、複製しています。 25 | Github URL: https://github.com/etcd-io/etcd 26 | ## Input／Output 27 | エッジクラスター内の各エッジノード端末において、titaniadbと連携して、データの制御、監視を行います。具体的には、titaniadbのデータを監視、取得、更新、追加、削除したり、ログ、エラー等のイベントの表示をしたりします。 28 | また、MySQLに静的な結果データのアップデートを行います。 29 | 30 | ## Install 31 | ``` 32 | $git clone git@bitbucket.org:latonaio/titaniadb-sentinel.git 33 | 34 | $cd titaniadb_sentinel 35 | 36 | $make docker-build 37 | ``` 38 | 39 | ## File Contents 40 | * etcd.py 41 | etcdに蓄積されたデータのkey、ID、metadataの取得や、それらのデータの挿入、追加、削除、に対するresponseを定義しています。 42 | Baseクラスでは、etcdにストアされたデータの監視、制御の開始、keyの取得、挿入、削除を定義しています。 43 | Deviceクラス、Podクラスでは、Baseクラスを継承し、それぞれDevice、Podに関するデータの操作を定義します。 44 | * main.py 45 | etcd.py、mysql.py で定義された内容に従って処理を実行し、各データベースのupsertや、処理内容、エラー内容の出力を行います。 46 | upsert_at_eventメソッドでは、eventのtypeに従って処理を行います。 47 | * mysql.py 48 | MySQLへの接続、カーソルの取得、データの操作など、データベース内のデータの操作方法を定義しています。 49 | BaseTableクラスは、データベースへの接続、カーソルの取得、データの取得などを定義しています。DeviceTableクラス、PodTbaleクラスでは、BaseTableクラスを継承し、それぞれDevice、Podに関するデータの操作を定義します。 50 | * deployment.yml 51 | deploymentは、PodとReplicaSetの宣言的なアップデート機能を提供するために作成されるyamlファイルです。 52 | * Dockerfile 53 | * Makefike 54 | * docker-build.sh 55 | * requirements.txt 56 | -------------------------------------------------------------------------------- /src/etcd.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import etcd3 5 | 6 | 7 | # Localhost 8 | HOST = os.environ.get('MY_ETCD_HOST', '0.0.0.0') 9 | PORT = int(os.environ.get('MY_ETCD_PORT', 2379)) 10 | TYPE_POSITION_INDEX = 1 11 | STATUS_POSITION_INDEX = 3 12 | RUNNINX_STATUS_INDEX = "0" 13 | STOPPED_STATUS_INDEX = "1" 14 | DEVICE_DATA_TYPE = "Device" 15 | POD_DATA_TYPE = "Pod" 16 | 17 | 18 | class Base(etcd3.Etcd3Client): 19 | def __init__(self): 20 | super().__init__(host=HOST, port=PORT) 21 | 22 | def watch_start(self): #prefixを持つkeyをwatchする. 23 | events_iter, self.cancel = self.watch_prefix(self.BASE_KEY) 24 | for event in events_iter: 25 | yield event 26 | return 27 | 28 | #keyは、/self.BASE_KEY/ID の形でストアされている. 29 | def get_dicts(self): #prefixを持つkeyのmetadata, valueを取得し、pod_data_typeなどの状態をdictsに入れる. 30 | tuples = self.get_prefix(self.BASE_KEY, sort_order='ascend') 31 | dicts = [] 32 | for b_value, b_metadata in tuples: 33 | value = b_value.decode('utf-8') 34 | _dict = json.loads(value) 35 | 36 | key = b_metadata.key.decode('utf-8') 37 | keyArr = key.split("/") 38 | if keyArr[STATUS_POSITION_INDEX] == STOPPED_STATUS_INDEX: 39 | if keyArr[TYPE_POSITION_INDEX] == DEVICE_DATA_TYPE: 40 | _dict["connectionStatus"] = 1 41 | elif keyArr[TYPE_POSITION_INDEX] == POD_DATA_TYPE: 42 | _dict["status"] = 1 43 | 44 | dicts.append(_dict) 45 | return dicts 46 | 47 | def get_dict(self, _id): 48 | key = f'{self.BASE_KEY}/{_id}' 49 | b_value, _ = self.get(key) 50 | if b_value is None: 51 | return b_value 52 | 53 | value = b_value.decode('utf-8') 54 | _dict = json.loads(value) 55 | return _dict 56 | 57 | def put_dict(self, _dict): 58 | _id = _dict[self.ID] 59 | key = f'{self.BASE_KEY}/{_id}' 60 | value = json.dumps(_dict) 61 | response = self.put(key, value) 62 | return response 63 | 64 | def delete(self, _dict): 65 | _id = _dict[self.ID] 66 | key = f'{self.BASE_KEY}/{_id}' 67 | response = super(Base, self).delete(key) 68 | return response 69 | 70 | 71 | class Device(Base): 72 | BASE_KEY = '/Device' 73 | ID = 'deviceName' 74 | 75 | def __init__(self): 76 | super().__init__() 77 | 78 | 79 | class Pod(Base): 80 | BASE_KEY = '/Pod' 81 | ID = 'podName' 82 | 83 | def __init__(self): 84 | super().__init__() 85 | -------------------------------------------------------------------------------- /src/examples/etcd.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import datetime 3 | 4 | from src import etcd 5 | from src.examples import data 6 | 7 | 8 | class Device(): 9 | def __init__(self): 10 | self.etcd_client = etcd.Device() 11 | 12 | def get(self): 13 | _dict = copy.deepcopy(data.device1) 14 | new_dict = self.etcd_client.get_dict(_dict[self.etcd_client.ID]) 15 | print(new_dict) 16 | return 17 | 18 | def put(self): 19 | _dict = copy.deepcopy(data.device1) 20 | _dict['deviceIp'] = datetime.datetime.now().isoformat() 21 | _dict['connectionStatus'] = datetime.datetime.now().timestamp() 22 | _dict['os'] = datetime.datetime.now().isoformat() 23 | res = self.etcd_client.put_dict(_dict) 24 | print(res) 25 | return 26 | 27 | def delete(self): 28 | _dict = copy.deepcopy(data.device1) 29 | res = self.etcd_client.delete(_dict) 30 | print(res) 31 | return 32 | 33 | def new(self): 34 | _dict = copy.deepcopy(data.device1) 35 | _dict['deviceName'] = datetime.datetime.now().isoformat() 36 | _dict['deviceIp'] = datetime.datetime.now().isoformat() 37 | _dict['connectionStatus'] = datetime.datetime.now().timestamp() 38 | _dict['os'] = datetime.datetime.now().isoformat() 39 | res = self.etcd_client.put_dict(_dict) 40 | print(res) 41 | return 42 | 43 | 44 | class Pod(): 45 | def __init__(self): 46 | self.etcd_client = etcd.Pod() 47 | 48 | def get(self): 49 | _dict = copy.deepcopy(data.pod1) 50 | new_dict = self.etcd_client.get_dict(_dict[self.etcd_client.ID]) 51 | print(new_dict) 52 | return 53 | 54 | def put(self): 55 | _dict = copy.deepcopy(data.pod1) 56 | _dict['currentVersion'] = datetime.datetime.now().isoformat() 57 | _dict['latestVersion'] = datetime.datetime.now().isoformat() 58 | _dict['deployedAt'] = datetime.datetime.now().isoformat() 59 | _dict['status'] = datetime.datetime.now().timestamp() 60 | res = self.etcd_client.put_dict(_dict) 61 | print(res) 62 | return 63 | 64 | def delete(self): 65 | _dict = copy.deepcopy(data.pod1) 66 | res = self.etcd_client.delete(_dict) 67 | print(res) 68 | return 69 | 70 | def new(self): 71 | _dict = copy.deepcopy(data.pod1) 72 | _dict['podName'] = datetime.datetime.now().isoformat() 73 | _dict['imageName'] = datetime.datetime.now().isoformat() 74 | _dict['currentVersion'] = datetime.datetime.now().isoformat() 75 | _dict['latestVersion'] = datetime.datetime.now().isoformat() 76 | _dict['deployedAt'] = datetime.datetime.now().isoformat() 77 | _dict['status'] = datetime.datetime.now().timestamp() 78 | res = self.etcd_client.put_dict(_dict) 79 | print(res) 80 | return 81 | 82 | 83 | if __name__ == "__main__": 84 | import argparse 85 | import sys 86 | 87 | parser = argparse.ArgumentParser() 88 | parser.add_argument('name', type=str) 89 | parser.add_argument('method', type=str) 90 | args = parser.parse_args() 91 | name = args.name 92 | method = args.method 93 | print(name, method) 94 | 95 | mapping = { 96 | 'device': Device, 97 | 'pod': Pod, 98 | } 99 | if mapping.get(name) is None: 100 | print(f'Not accept name: {name}') 101 | sys.exit(1) 102 | obj = mapping.get(name)() 103 | 104 | methods = { 105 | 'g': 'get', 106 | 'p': 'put', 107 | 'd': 'delete', 108 | 'n': 'new', 109 | } 110 | if methods.get(method) is None: 111 | print(f'Not accept method: {method}') 112 | sys.exit(1) 113 | new_method = methods.get(method) 114 | 115 | eval(f'obj.{new_method}')() 116 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import queue 4 | import threading 5 | import time 6 | import pprint 7 | import traceback 8 | 9 | from aion.logger import lprint, initialize_logger 10 | from mysql import DeviceTable as deviceTable, PodTable as podTable 11 | from etcd import Device as etcdDevice, Pod as etcdPod 12 | from etcd3.events import PutEvent, DeleteEvent 13 | 14 | #環境変数の取得 15 | SERVICE_NAME = os.environ.get("SERVICE", "titaniadb-sentinel") 16 | DEVICE_INTERVAL = os.environ.get("DEVICE_INTERVAL", "30") 17 | POD_INTERVAL = os.environ.get("POD_INTERVAL", "5") 18 | initialize_logger(SERVICE_NAME) 19 | 20 | 21 | def upsert_at_event(etcd_client_class, mysql_client_class, _queue): 22 | try: 23 | lprint(f'etcd Watch {etcd_client_class.BASE_KEY} key prefix.') 24 | etcd_client = etcd_client_class() 25 | for event in etcd_client.watch_start(): #watch_start()はetcd.pyにて定義.接頭辞のあるキーの変更を取得する. 26 | key = None 27 | value = None 28 | key = event.key.decode('utf-8') 29 | value = event.value.decode('utf-8') 30 | 31 | if type(event) == PutEvent: 32 | error_count = 0 33 | while True: 34 | try: 35 | error_count += 1 36 | lprint(f'etcd get Put Event: {key}') 37 | _dict = json.loads(value) 38 | with mysql_client_class() as mysql_client: 39 | mysql_client.upsert(_dict) 40 | lprint(f'mysql upsert: {key}') #Put Eventを検知し、Valueを書き換える. 41 | break 42 | except Exception as e: 43 | if error_count < 5: 44 | lprint(f'Error was occurred at {error_count} times for {key} key. Retry it after sleep.') 45 | time.sleep(2) #Error発生(５回以内)時の表示. 46 | else: 47 | lprint('=' * 50) 48 | lprint(f'event: {event.__class__.__name__}') 49 | lprint(f'key: {key}') 50 | lprint(f'value:') 51 | lprint(value) 52 | lprint('=' * 50) 53 | raise e #５回以上Errorが発生した場合の表示.Error内容と発生箇所を表示する. 54 | 55 | elif type(event) == DeleteEvent: 56 | lprint(f'etcd get Delete Event: {key}') 57 | 58 | else: 59 | raise RuntimeError(f'etcd get unexpected event: {event}') 60 | 61 | except Exception as e: 62 | lprint(e) 63 | message = f'etcd finish to Watch {etcd_client_class.BASE_KEY} key prefix.' 64 | _queue.put(message) 65 | return 66 | 67 | return 68 | 69 | 70 | def upsert_all(etcd_client_class, mysql_client_class, interval, _queue): #すべてのデータの更新/挿入を行う. 71 | try: 72 | while True: 73 | etcd_client = etcd_client_class() 74 | dicts = etcd_client.get_dicts() #get_dicts()はetcd.pyにて定義.get_prefixで取得されたデータが取得される. 75 | with mysql_client_class() as mysql_client: 76 | for _dict in dicts: 77 | mysql_client.upsert(_dict) 78 | 79 | del etcd_client 80 | 81 | lprint(f'Success to upsert {mysql_client_class.__name__}.') 82 | pprint.pprint(dicts) 83 | time.sleep(int(interval)) 84 | 85 | except Exception: 86 | lprint('=' * 50) 87 | lprint(traceback.format_exc()) 88 | message = f'Faild to upsert all. mysql_client_class class is {mysql_client_class.__name__}.' 89 | _queue.put(message) 90 | return 91 | 92 | 93 | def main(): 94 | _queue = queue.Queue() 95 | 96 | thread_device_event = threading.Thread(target=upsert_at_event, args=(etcdDevice, deviceTable, _queue), daemon=True) 97 | thread_pod_event = threading.Thread(target=upsert_at_event, args=(etcdPod, podTable, _queue), daemon=True) 98 | thread_device_interval = threading.Thread(target=upsert_all, args=(etcdDevice, deviceTable, DEVICE_INTERVAL, _queue), daemon=True) 99 | thread_pod_interval = threading.Thread(target=upsert_all, args=(etcdPod, podTable, POD_INTERVAL, _queue), daemon=True) 100 | 101 | thread_device_event.start() 102 | thread_pod_event.start() 103 | time.sleep(5) 104 | thread_device_interval.start() 105 | thread_pod_interval.start() 106 | 107 | message = _queue.get() 108 | lprint('=' * 50) 109 | lprint(message) 110 | lprint('Cancel all threads.') 111 | return 112 | 113 | 114 | if __name__ == "__main__": 115 | main() 116 | -------------------------------------------------------------------------------- /src/mysql.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | 5 | import MySQLdb 6 | 7 | 8 | # Localhost 9 | HOST = os.environ.get('MY_MYSQL_HOST', '127.0.0.1') 10 | PORT = int(os.environ.get('MY_MYSQL_PORT', "3306")) 11 | USER = os.environ.get('MY_MYSQL_USER') 12 | PASSWORD = os.environ.get('MY_MYSQL_PASSWORD') 13 | 14 | # re compile at first 15 | re_eof = re.compile(r'\n') 16 | re_space = re.compile(r'\s+') 17 | 18 | 19 | def format_sql(sql): 20 | sql = re_eof.sub(' ', sql) 21 | sql = re_space.sub(' ', sql) 22 | if sys.flags.debug: 23 | print(sql) 24 | return sql 25 | 26 | 27 | class BaseTable(): 28 | def __enter__(self): #DBに接続し、カーソルを取得する. 29 | try: 30 | self.connection = MySQLdb.connect( 31 | host=HOST, port=PORT, user=USER, passwd=PASSWORD, charset='utf8') 32 | self.cursor = self.connection.cursor(MySQLdb.cursors.DictCursor) 33 | except MySQLdb.Error as e: 34 | print('Faild to connect MySQL Server.') 35 | raise e 36 | return self 37 | 38 | def __exit__(self, exception_type, exception_value, traceback): #カーソルを閉じる 39 | if self.cursor: 40 | self.cursor.close() 41 | self.connection.close() 42 | return 43 | 44 | def is_connect(self): 45 | return bool(self.cursor) 46 | 47 | def get_query(self, sql, args=None): #一件ずつデータを取得する 48 | if not self.cursor: 49 | return None 50 | self.cursor.execute(sql, args) 51 | return self.cursor.fetchone() 52 | 53 | def get_query_list(self, sql, size, args=None): 54 | if not self.cursor: 55 | return None 56 | self.cursor.execute(sql, args) 57 | return self.cursor.fetchmany(size) 58 | 59 | def set_query(self, sql, args=None): 60 | if not self.cursor: 61 | return False 62 | self.cursor.execute(sql, args) 63 | return True 64 | 65 | def commit_query(self): 66 | self.connection.commit() 67 | return 68 | 69 | 70 | class DeviceTable(BaseTable): 71 | DATABASE_NAME = 'Device' 72 | TABLE_NAME = 'device' 73 | 74 | def select(self, deviceName): 75 | sql = f''' 76 | SELECT * 77 | FROM {self.DATABASE_NAME}.{self.TABLE_NAME} 78 | WHERE deviceName = '{deviceName}'; 79 | ''' 80 | sql = format_sql(sql) 81 | _dict = self.get_query(sql) 82 | return _dict 83 | 84 | def upsert(self, _dict): 85 | sql = f""" 86 | INSERT INTO {self.DATABASE_NAME}.{self.TABLE_NAME} ( 87 | deviceName, projectSymbolFk, deviceIp, connectionStatus, os 88 | ) VALUES ( 89 | '{_dict['deviceName']}', '{_dict['projectSymbolFk']}', 90 | '{_dict['deviceIp']}', '{_dict['connectionStatus']}', '{_dict['os']}' 91 | ) 92 | ON DUPLICATE KEY UPDATE 93 | deviceName = '{_dict['deviceName']}', 94 | projectSymbolFk = '{_dict['projectSymbolFk']}', 95 | deviceIp = '{_dict['deviceIp']}', 96 | connectionStatus = '{_dict['connectionStatus']}', 97 | os = '{_dict['os']}'; 98 | """ 99 | sql = format_sql(sql) 100 | self.set_query(sql) 101 | self.commit_query() 102 | return 103 | 104 | def delete(self): 105 | sql = f""" 106 | DELETE FROM {self.DATABASE_NAME}.{self.TABLE_NAME} 107 | """ 108 | sql = format_sql(sql) 109 | self.set_query(sql) 110 | self.commit_query() 111 | return 112 | 113 | 114 | class PodTable(BaseTable): 115 | DATABASE_NAME = 'Pod' 116 | TABLE_NAME = 'pod' 117 | 118 | def upsert(self, _dict): 119 | # imageName 120 | imageName = _dict.get('imageName') 121 | if imageName is not None: 122 | imageName = f"'{imageName}'" 123 | else: 124 | imageName = 'NULL' 125 | # deployedAt 126 | if _dict['deployedAt']: 127 | deployedAt = f"'{_dict['deployedAt']}'" 128 | else: 129 | deployedAt = 'NULL' 130 | 131 | sql = f""" 132 | INSERT INTO {self.DATABASE_NAME}.{self.TABLE_NAME} ( 133 | podName, deviceNameFk, imageName, currentVersion, latestVersion, deployedAt, status 134 | ) VALUES ( 135 | '{_dict['podName']}', '{_dict['deviceNameFk']}', {imageName}, 136 | '{_dict['currentVersion']}', '{_dict['latestVersion']}', 137 | {deployedAt}, '{_dict['status']}' 138 | ) 139 | ON DUPLICATE KEY UPDATE 140 | podName = '{_dict['podName']}', 141 | deviceNameFk = '{_dict['deviceNameFk']}', 142 | imageName = {imageName}, 143 | currentVersion = '{_dict['currentVersion']}', 144 | latestVersion = '{_dict['latestVersion']}', 145 | deployedAt = {deployedAt}, 146 | status = '{_dict['status']}'; 147 | """ 148 | sql = format_sql(sql) 149 | self.set_query(sql) 150 | self.commit_query() 151 | return 152 | 153 | def delete(self): 154 | sql = f""" 155 | DELETE FROM {self.DATABASE_NAME}.{self.TABLE_NAME} 156 | """ 157 | sql = format_sql(sql) 158 | self.set_query(sql) 159 | self.commit_query() 160 | return 161 | --------------------------------------------------------------------------------