├── .gitignore ├── LICENSE ├── SoftwareDefinedCache.py └── SDCManager.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | *.pyc 10 | 11 | # Packages # 12 | ############ 13 | # it's better to unpack these files and commit the raw source 14 | # git has its own built in compression methods 15 | *.7z 16 | *.dmg 17 | *.gz 18 | *.iso 19 | *.rar 20 | #*.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sqlite 27 | *.xml 28 | _windows/ 29 | 30 | # OS generated files # 31 | ###################### 32 | .DS_Store 33 | ehthumbs.db 34 | Icon 35 | Thumbs.db 36 | .tmtags 37 | .idea/ 38 | mydjangosite/.idea/ 39 | tags 40 | vendor.tags 41 | tmtagsHistory 42 | *.sublime-project 43 | *.sublime-workspace 44 | .bundle 45 | 46 | cache/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2019, Intelligent-distributed Cloud and Security Laboratory (ICNS Lab.) 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /SoftwareDefinedCache.py: -------------------------------------------------------------------------------- 1 | # This file is part of Qualified Caching-as-a-Service. 2 | # BSD 3-Clause License 3 | # 4 | # Copyright (c) 2019, Intelligent-distributed Cloud and Security Laboratory (ICNS Lab.) 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are met: 9 | # 10 | # * Redistributions of source code must retain the above copyright notice, this 11 | # list of conditions and the following disclaimer. 12 | # 13 | # * Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # 17 | # * Neither the name of the copyright holder nor the names of its 18 | # contributors may be used to endorse or promote products derived from 19 | # this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | # title : SoftwareDefinedCache.py 33 | # description : python SoftwareDefinedCache class 34 | # author : Yunkon(Alvin) Kim 35 | # date : 20190130 36 | # version : 0.1 37 | # python_version : 3.6 38 | # notes : This SoftwareDefinedCache is an implementation of a cache managed by software 39 | # in the Python Programming Language. 40 | # ============================================================================== 41 | from datetime import datetime 42 | import os 43 | import sys 44 | import threading 45 | import time 46 | import json 47 | 48 | 49 | class SoftwareDefinedCache: 50 | 51 | def __init__(self, directory, capacity, data_retention_period=900): 52 | """initialize this class 53 | 54 | :param directory: the dir to create files for cache 55 | :param capacity: the capacity of the cache (unit: MB) 56 | :param data_retention_period: the data retention period to delete data (unit: second) 57 | """ 58 | self._lock = threading.Lock() 59 | self._used_lock = threading.Lock() 60 | self._buffer_lock = threading.Lock() 61 | self._is_not_full_cache = threading.Condition() 62 | self._is_not_empty_buffer = threading.Condition() 63 | self.directory = directory 64 | self.capacity = (capacity << 20) 65 | self._used = 0 66 | self.data_retention_period = data_retention_period 67 | self.seek_start_point = 0 68 | self._buffer = bytearray() 69 | 70 | def initialize(self): 71 | self._used = 0 72 | self.seek_start_point = 0 73 | 74 | def put_to_write_buffer(self, data): 75 | with self._is_not_empty_buffer: 76 | # print("Type of data: %s" % type(data)) 77 | self._buffer_lock.acquire() 78 | self.buffer.extend(data) 79 | print("Buffer len: %s" % len(self.buffer)) 80 | self._buffer_lock.release() 81 | if len(self.buffer) > 0: 82 | self._is_not_empty_buffer.notify() 83 | 84 | def store_data_with_write_buffer(self): 85 | print("Thread start - store data with write-buffer") 86 | with self._is_not_empty_buffer: 87 | while True: 88 | self._is_not_empty_buffer.wait() 89 | 90 | with self._is_not_full_cache: 91 | if len(self.buffer) > 0: 92 | remaining_capacity = self.capacity - self.used 93 | if remaining_capacity > 0: 94 | self._buffer_lock.acquire() 95 | data = self.buffer[:remaining_capacity] 96 | self.buffer = self.buffer[remaining_capacity:] 97 | self._buffer_lock.release() 98 | self.store_data(data) 99 | print("Stored data length: %s" % format(len(data), ",")) 100 | elif remaining_capacity == 0: # Cache is full 101 | self._is_not_full_cache.wait() 102 | else: 103 | print("Unknown process (remaining_capacity < 0)") 104 | elif len(self.buffer) == 0: 105 | print("Buffer is empty! (%s)" % self.buffer) 106 | else: 107 | print("Unknown (Buffer is lower than 0)") 108 | 109 | # (Not covered here) recv data <- SDC Manager is in charge of communication with EDCrammer or a cloud service. 110 | # store data to cache 111 | def store_data(self, data): 112 | # file name should includes milliseconds because caching is occurred quickly. 113 | # [:-3] means convert microseconds to milliseconds 114 | self._lock.acquire() 115 | file_name = datetime.now().strftime("%Y%m%d%H%M%S.%f")[:-3] # time.strftime("%Y%m%d%H%M%S") 116 | full_path = os.path.join(self.directory, file_name) 117 | 118 | f = open(full_path, 'wb') 119 | f.write(data) 120 | 121 | # read file size and convert byte to megabyte 122 | self.used += os.path.getsize(full_path) 123 | 124 | f.close() 125 | self._lock.release() 126 | 127 | # !!! the other function to read data is necessary. (e.g., "index.html" file it self) it depends on a service. 128 | # read first data from a directory 129 | def read_first_data(self): 130 | data = False 131 | if self.used > 0: 132 | self._lock.acquire() 133 | 134 | files = sorted(os.listdir(self.directory)) 135 | if len(files) > 0: 136 | file_name = files.pop(0) 137 | full_path = os.path.join(self.directory, file_name) 138 | 139 | f = open(full_path, 'rb') 140 | 141 | data = f.read() 142 | f.close() 143 | 144 | self.used -= os.path.getsize(full_path) 145 | os.remove(full_path) 146 | 147 | self._lock.release() 148 | 149 | return data 150 | 151 | def read_bytes(self, size): 152 | data = b'' 153 | result = False 154 | print("Utilization: %s" % self.used) 155 | try: 156 | 157 | if self.used >= size: 158 | with self._is_not_full_cache: 159 | self._lock.acquire() 160 | # if used is bigger or equal with size, the below 2 lines are naturally passed. 161 | # files = sorted(os.listdir(self.directory)) 162 | # if len(files) > 0: 163 | data_size = 0 164 | 165 | while data_size < size: 166 | files = sorted(os.listdir(self.directory)) 167 | # always pop 0 because the read file is deleted when seek_pointer reaches EOF. 168 | file_name = files.pop(0) 169 | full_path = os.path.join(self.directory, file_name) 170 | 171 | f = open(full_path, 'rb') 172 | 173 | f.seek(self.seek_start_point) 174 | read_data = f.read(size - data_size) 175 | 176 | f.close() 177 | 178 | read_data_size = len(read_data) 179 | file_size = os.path.getsize(full_path) 180 | self.used -= read_data_size 181 | 182 | # update seek_start_point for future access 183 | self.seek_start_point += read_data_size 184 | 185 | # print("Size: %s /// data_size: %s" % 186 | # (size, data_size)) 187 | 188 | # print("Used: %s /// seek_start_point: %s /// file_size: %s" % 189 | # (self.used, self.seek_start_point, file_size)) 190 | 191 | if file_size <= self.seek_start_point: 192 | # print("!!!!!!!!!!!!!!!!DELETE %s" % full_path) 193 | os.remove(full_path) 194 | self.seek_start_point = 0 195 | 196 | data += read_data 197 | data_size += read_data_size 198 | 199 | result = True 200 | self._lock.release() 201 | self._is_not_full_cache.notify() 202 | except Exception as e: 203 | result = False 204 | print("!!!!! Exception: %s" % e) 205 | 206 | return data, result 207 | 208 | def decay_data(self): 209 | """Running as a thread, this function deletes data considering last data hit. 210 | (e.g., certain period of time has passed since the last hit.) 211 | """ 212 | # aging checking function will be added. 213 | current_time = time.time() 214 | 215 | self._lock.acquire() 216 | 217 | files = sorted(os.listdir(self.directory)) 218 | for file_name in files: 219 | full_path = os.path.join(self.directory, file_name) 220 | no_hit_period = current_time - os.stat(full_path).st_atime 221 | if no_hit_period > self.data_retention_period: 222 | os.remove(full_path) 223 | 224 | self._lock.release() 225 | 226 | def run(self): 227 | # t1 = threading.Thread(target=self.decay_data) 228 | t1 = threading.Thread(target=self.store_data_with_write_buffer) 229 | t1.start() 230 | # t1.join() 231 | 232 | # for monitoring cache status 233 | def get_cache_status(self): 234 | cache_status = { 235 | "directory": self.directory, 236 | "capacity": self.capacity, 237 | "used": self.used, 238 | "available": self.capacity-self.used, 239 | "data_retention_period": self.data_retention_period 240 | } 241 | return json.dumps(cache_status) 242 | 243 | @property 244 | def used(self): 245 | self._used_lock.acquire() 246 | val = self._used 247 | self._used_lock.release() 248 | return val 249 | 250 | @used.setter 251 | def used(self, val): 252 | self._used_lock.acquire() 253 | self._used = val 254 | self._used_lock.release() 255 | 256 | @property 257 | def buffer(self): 258 | return self._buffer 259 | 260 | @buffer.setter 261 | def buffer(self, buffer): 262 | self._buffer = buffer 263 | -------------------------------------------------------------------------------- /SDCManager.py: -------------------------------------------------------------------------------- 1 | # This file is part of Qualified Caching-as-a-Service. 2 | # BSD 3-Clause License 3 | # 4 | # Copyright (c) 2019, Intelligent-distributed Cloud and Security Laboratory (ICNS Lab.) 5 | # All rights reserved. 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions are met: 9 | # 10 | # * Redistributions of source code must retain the above copyright notice, this 11 | # list of conditions and the following disclaimer. 12 | # 13 | # * Redistributions in binary form must reproduce the above copyright notice, 14 | # this list of conditions and the following disclaimer in the documentation 15 | # and/or other materials provided with the distribution. 16 | # 17 | # * Neither the name of the copyright holder nor the names of its 18 | # contributors may be used to endorse or promote products derived from 19 | # this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | # 32 | # title : SoftwareDefinedCache.py 33 | # description : python SDCManager class 34 | # author : Yunkon(Alvin) Kim 35 | # date : 20190130 36 | # version : 0.1 37 | # python_version : 3.6 38 | # notes : This class is an implementation of a manager to handle SDC 39 | # in the Python Programming Language. 40 | # ============================================================================== 41 | 42 | import logging 43 | import os 44 | # import logging 45 | import random 46 | import threading 47 | import time 48 | 49 | import paho.mqtt.client as mqtt 50 | from paho.mqtt import publish 51 | 52 | import SoftwareDefinedCache as SDC 53 | 54 | rootlogger = logging.getLogger(__name__) 55 | rootlogger.setLevel("DEBUG") 56 | 57 | SDC_id = "SDC_1" 58 | client_id = "Client_1" 59 | 60 | sdc = SDC.SoftwareDefinedCache(os.path.join(".", "cache"), 5) 61 | 62 | MQTT_HOST = "163.180.117.236" 63 | MQTT_PORT = 1883 64 | 65 | # MQTT_HOST_ON_EDGE = "192.168.0.58" 66 | # MQTT_PORT_ON_EDGE = 1883 67 | MQTT_HOST_ON_EDGE = "163.180.117.185" 68 | MQTT_PORT_ON_EDGE = 11883 69 | 70 | # ----------------------------------------Error calculation for PID controller---------------------------------------# 71 | # Assume 90% cache utilization 72 | TARGET_UTILIZATION = 0.9 73 | 74 | TEST_TIME = 30 # sec 75 | 76 | is_testing = True 77 | 78 | conditionLock = threading.Condition() 79 | 80 | flow_control_delay = 0.03 81 | 82 | 83 | def calculate_error(target, current): 84 | return target - current 85 | 86 | 87 | # -------------------------------------------------------------------------------------------------------------------# 88 | 89 | # ---------------------------------------------------- Producer ---------------------------------------------------- # 90 | def notify_storage_status(): 91 | # how long? many? 92 | global conditionLock 93 | try: 94 | start_time = time.time() 95 | with conditionLock: 96 | while True: 97 | # calculate error (utilization difference) 98 | # error = calculate_error(sdc.capacity * TARGET_UTILIZATION, sdc.used) 99 | # print("error :%s" % error) 100 | # error = calculate_error(sdc.capacity * TARGET_UTILIZATION, sdc.used) 101 | print("feedback :%s" % sdc.used) 102 | # send feedback 103 | publish.single("core/edge/" + SDC_id + "/feedback", sdc.used, hostname=MQTT_HOST, port=MQTT_PORT, qos=2) 104 | conditionLock.wait() 105 | time.sleep(0.03) 106 | running_time = time.time() - start_time 107 | if running_time > TEST_TIME: 108 | break 109 | except Exception as e: 110 | print("Exception: %s" % e) 111 | 112 | 113 | # Creating threads 114 | 115 | # # ----------------------------------------------------RESTful API----------------------------------------------------# 116 | # app = Flask(__name__) 117 | # api = Api(app) 118 | # 119 | # 120 | # class Introduction: 121 | # def get(self): 122 | # introduction = """ 123 | # Hello! 124 | # This is the RESTful API for Software-Defined Cache. 125 | # By "/help", you can see the list of Method(Create, Read, Update, Delete) and Resources(URI). 126 | # """ 127 | # return introduction 128 | # 129 | # 130 | # class Help(Resource): 131 | # def get(self): 132 | # help_message = """ 133 | # API Usage: 134 | # - GET / 135 | # - GET /help 136 | # - GET /api/data/ 137 | # - GET /api/data/ 138 | # """ 139 | # return help_message 140 | # 141 | # 142 | # class CachedData(Resource): 143 | # def get(self, data_id): 144 | # if not data_id: 145 | # print('First In First Out!') 146 | # data = sdc.read_first_data() 147 | # else: 148 | # print('The data out') 149 | # # read and return 150 | # 151 | # return data 152 | # 153 | # 154 | # api.add_resource(Introduction, '/') 155 | # api.add_resource(Help, '/help') 156 | # api.add_resource(CachedData, '/api/data/', '/api/data/') 157 | # 158 | # 159 | # # -------------------------------------------------------------------------------------------------------------------# 160 | 161 | 162 | # --------------------------------------------------MQTT Core-Edge----------------------------------------------------# 163 | def on_connect(client, userdata, flags, rc): 164 | if rc == 0: 165 | print("Connected to central MQTT broker - Result code: " + str(rc)) 166 | client.subscribe("core/edge/" + SDC_id + "/data") 167 | client.subscribe("core/edge/" + SDC_id + "/flow_control") 168 | client.subscribe("core/edge/" + SDC_id + "/start_testing") 169 | 170 | else: 171 | print("Bad connection returned code = ", rc) 172 | print("ERROR: Could not connect to MQTT") 173 | 174 | 175 | def on_message(client, userdata, msg): 176 | global conditionLock 177 | global flow_control_delay 178 | 179 | try: 180 | # print("Cart new message: " + msg.topic + " " + str(msg.payload)) 181 | message = msg.payload 182 | print("Arrived topic: %s" % msg.topic) 183 | # print("Arrived message: %s" % message) 184 | 185 | if msg.topic == "core/edge/" + SDC_id + "/data": 186 | with conditionLock: 187 | # sdc.store_data(message) 188 | sdc.put_to_write_buffer(message) 189 | print("Data size: %s" % len(message)) 190 | # data_size = int(message) 191 | # print("Data size: %s" % data_size) 192 | # sdc.used += data_size 193 | flow_control_delay = 0.03 194 | conditionLock.notify() 195 | # error = calculate_error(sdc.capacity * TARGET_UTILIZATION, sdc.used) 196 | # print("error: %s" % error) 197 | # time.sleep(0.03) 198 | # print("used: %s" % sdc.used) 199 | # publish.single("core/edge/" + SDC_id + "/feedback", sdc.used, hostname=MQTT_HOST, port=MQTT_PORT) 200 | elif msg.topic == "core/edge/" + SDC_id + "/flow_control": 201 | with conditionLock: 202 | print("~~~~Flow_control~~~~") 203 | time.sleep(0.03) 204 | flow_control_delay += 0.03 205 | conditionLock.notify() 206 | # error = calculate_error(sdc.capacity * TARGET_UTILIZATION, sdc.used) 207 | # print("error: %s" % error) 208 | # print("used: %s" % sdc.used) 209 | # time.sleep(0.03) 210 | # publish.single("core/edge/" + SDC_id + "/feedback", sdc.used, hostname=MQTT_HOST, port=MQTT_PORT) 211 | elif msg.topic == "core/edge/" + SDC_id + "/start_testing": 212 | print("Start testing!!") 213 | storage_status_notifying = threading.Thread(target=notify_storage_status) 214 | storage_status_notifying.start() 215 | time.sleep(0.03) 216 | # publish.single("core/edge/" + SDC_id + "/feedback", sdc.used, hostname=MQTT_HOST, port=MQTT_PORT) 217 | publish.single("edge/client/" + client_id + "/start_caching", 1, hostname=MQTT_HOST_ON_EDGE, 218 | port=MQTT_PORT_ON_EDGE, qos=2) 219 | else: 220 | print("Unknown - topic: " + msg.topic + ", message: " + message) 221 | except Exception as e: 222 | print("Exception: %s" % e) 223 | 224 | 225 | def on_publish(client, userdata, mid): 226 | print("mid: " + str(mid)) 227 | 228 | 229 | def on_subscribe(client, userdata, mid, granted_qos): 230 | print("Subscribed: " + str(mid) + " " + str(granted_qos)) 231 | 232 | 233 | def on_log(client, userdata, level, string): 234 | print(string) 235 | 236 | 237 | # The below lines will be used to publish the topics 238 | # publish.single("elevator/starting_floor_number", "3", hostname=MQTT_HOST, port=MQTT_PORT) 239 | # publish.single("elevator/destination_floor_number", "2", hostname=MQTT_HOST, port=MQTT_PORT) 240 | # ------------------------------------------------------------------------------------------------------------------# 241 | 242 | # --------------------------------------------------MQTT Edge-Client---------------------------------------------------# 243 | def on_local_connect(client, userdata, flags, rc): 244 | if rc == 0: 245 | print("Connected to local MQTT broker - Result code: " + str(rc)) 246 | client.subscribe("edge/client/" + client_id + "/data_req") 247 | client.subscribe("edge/client/" + client_id + "/done_to_test") 248 | 249 | else: 250 | print("Bad connection returned code = ", rc) 251 | print("ERROR: Could not connect to MQTT") 252 | 253 | 254 | def on_local_message(client, userdata, msg): 255 | global is_testing 256 | 257 | try: 258 | # print("Cart new message: " + msg.topic + " " + str(msg.payload)) 259 | message = msg.payload 260 | print("Arrived topic: %s" % msg.topic) 261 | print("Arrived message: %s" % message) 262 | 263 | if msg.topic == "edge/client/" + client_id + "/data_req": 264 | read_size = int(message) 265 | print("Requested amount of data: %s" % read_size) 266 | data, result = sdc.read_bytes(read_size) 267 | # 268 | # if sdc.used < read_size: 269 | # read_size = sdc.used 270 | # sdc.used = 0 271 | # else: 272 | # sdc.used -= read_size 273 | 274 | if result: 275 | print('*** ', end='', flush=True) 276 | print("Length of transmitted data: %s" % len(data)) 277 | publish.single("edge/client/" + client_id + "/data", data, hostname=MQTT_HOST_ON_EDGE, 278 | port=MQTT_PORT_ON_EDGE, qos=2) 279 | else: 280 | print("Cache misses (no data or not enough data)") 281 | publish.single("edge/client/" + client_id + "/data", False, hostname=MQTT_HOST_ON_EDGE, 282 | port=MQTT_PORT_ON_EDGE, qos=2) 283 | 284 | elif msg.topic == "edge/client/" + client_id + "/done_to_test": 285 | publish.single("core/edge/" + SDC_id + "/done_to_test", "done", hostname=MQTT_HOST, port=MQTT_PORT, qos=2) 286 | time.sleep(3) 287 | is_testing = False 288 | else: 289 | print("Unknown - topic: " + msg.topic + ", message: " + message) 290 | except Exception as e: 291 | print("Exception: %s" % e) 292 | 293 | 294 | def on_local_publish(client, userdata, mid): 295 | print("mid: " + str(mid)) 296 | 297 | 298 | def on_local_subscribe(client, userdata, mid, granted_qos): 299 | print("Subscribed: " + str(mid) + " " + str(granted_qos)) 300 | 301 | 302 | def on_local_log(client, userdata, level, string): 303 | print(string) 304 | 305 | 306 | # The below lines will be used to publish the topics 307 | # publish.single("elevator/starting_floor_number", "3", hostname=MQTT_HOST, port=MQTT_PORT) 308 | # publish.single("elevator/destination_floor_number", "2", hostname=MQTT_HOST, port=MQTT_PORT) 309 | # ------------------------------------------------------------------------------------------------------------------# 310 | 311 | 312 | # ---------------------------------------------------- Consumer ---------------------------------------------------- # 313 | # def consume_data_scenario1(): 314 | # # A cloud service periodically consumes an equal amount of cached data. 315 | # 316 | # start_time = time.time() 317 | # read_size = (2 << 19) 318 | # while True: 319 | # # consume data 320 | # # This section will be changed to apply the distributed messaging structure. 321 | # # In other words, MQTT will be used. 322 | # sdc.read_bytes(read_size) 323 | # # print("Consuming data") 324 | # running_time = time.time() - start_time 325 | # if running_time > TEST_TIME: 326 | # break 327 | # time.sleep(0.03) 328 | # 329 | # 330 | # def consume_data_scenario2(): 331 | # # A cloud service periodically consumes an unequal amount of cached data. 332 | # 333 | # start_time = time.time() 334 | # # read_size = (4 << 19) 335 | # while True: 336 | # # This section will be changed to apply the distributed messaging structure. 337 | # val = random.randint(1, 2) 338 | # val2 = random.randint(16, 19) 339 | # read_size = (val << val2) 340 | # # In other words, MQTT will be used. 341 | # sdc.read_bytes(read_size) 342 | # # print("Consuming data") 343 | # running_time = time.time() - start_time 344 | # if running_time > TEST_TIME: 345 | # break 346 | # time.sleep(0.03) 347 | # 348 | # 349 | # def consume_data_scenario3(): 350 | # # A cloud service aperiodically consumes an equal amount of cached data. 351 | # 352 | # start_time = time.time() 353 | # read_size = (2 << 19) 354 | # while True: 355 | # # This section will be changed to apply the distributed messaging structure. 356 | # # In other words, MQTT will be used. 357 | # random_ms = random.randint(3, 100) / 1000.0 358 | # sdc.read_bytes(read_size) 359 | # # print("Consuming data") 360 | # running_time = time.time() - start_time 361 | # if running_time > TEST_TIME: 362 | # break 363 | # time.sleep(random_ms) 364 | # 365 | # 366 | # def consume_data_scenario4(): 367 | # # A cloud service aperiodically consumes an unequal amount of cached data. 368 | # 369 | # start_time = time.time() 370 | # while True: 371 | # # This section will be changed to apply the distributed messaging structure. 372 | # val = random.randint(1, 2) 373 | # val2 = random.randint(16, 19) 374 | # random_ms = random.randint(3, 100) / 1000.0 375 | # read_size = (val << val2) 376 | # # In other words, MQTT will be used. 377 | # sdc.read_bytes(read_size) 378 | # # print("Consuming data") 379 | # running_time = time.time() - start_time 380 | # if running_time > TEST_TIME: 381 | # break 382 | # time.sleep(random_ms) 383 | 384 | 385 | def main(): 386 | global is_testing 387 | # RESTful API runs 388 | # app.run(debug=True) 389 | 390 | # MQTT connection 391 | message_client = mqtt.Client("Edge1") 392 | message_client.on_connect = on_connect 393 | message_client.on_message = on_message 394 | # message_client.on_log = on_log 395 | # Connect to MQTT broker 396 | # message_client.connect(MQTT_HOST, MQTT_PORT, 60) 397 | # message_client.loop_start() 398 | 399 | # MQTT connection 400 | message_local_client = mqtt.Client("Edge2") 401 | message_local_client.on_connect = on_local_connect 402 | message_local_client.on_message = on_local_message 403 | # message_local_client.on_log = on_local_log 404 | 405 | # message_local_client.connect(MQTT_HOST_ON_EDGE, MQTT_PORT_ON_EDGE, 60) 406 | # message_local_client.loop_start() 407 | 408 | # Software-Defined Cache runs 409 | sdc.run() 410 | print("SDC runs") 411 | scenario_counter = 1 412 | 413 | while scenario_counter <= 4: 414 | 415 | # preparation time 416 | time.sleep(5) 417 | 418 | loop_counter = 0 419 | loop_round = 9 420 | 421 | while loop_counter < loop_round: 422 | # ---------- Scenario 1 423 | 424 | print("Truncate cache") 425 | directory = os.path.join(".", "cache") 426 | files = sorted(os.listdir(directory)) 427 | for file_name in files: 428 | full_path = os.path.join(directory, file_name) 429 | os.remove(full_path) 430 | 431 | print("Start testing") 432 | time.sleep(3) 433 | 434 | sdc.initialize() 435 | 436 | message_client.connect(MQTT_HOST, MQTT_PORT, 60) 437 | message_client.loop_start() 438 | message_local_client.connect(MQTT_HOST_ON_EDGE, MQTT_PORT_ON_EDGE, 60) 439 | message_local_client.loop_start() 440 | 441 | publish.single("core/edge/" + SDC_id + "/init_for_testing", scenario_counter, hostname=MQTT_HOST, 442 | port=MQTT_PORT, qos=2) 443 | 444 | # # Creating threads 445 | # t1 = threading.Thread(target=consume_data_scenario1) 446 | # # Starting threads 447 | # t1.start() 448 | # # Wait until threads are completely executed 449 | # t1.join() 450 | # print("Test 1 is done!") 451 | while is_testing: 452 | time.sleep(0.005) 453 | 454 | print("Done to test") 455 | message_client.loop_stop() 456 | message_client.disconnect() 457 | message_local_client.loop_stop() 458 | message_local_client.disconnect() 459 | 460 | is_testing = True 461 | 462 | # time.sleep(2) 463 | # 464 | # message_client.connect(MQTT_HOST, MQTT_PORT, 60) 465 | # message_client.loop_start() 466 | # 467 | # # ---------- Scenario 2 468 | # 469 | # print("Truncate cache") 470 | # directory = os.path.join(".", "cache") 471 | # files = sorted(os.listdir(directory)) 472 | # for file_name in files: 473 | # full_path = os.path.join(directory, file_name) 474 | # os.remove(full_path) 475 | # 476 | # print("Start testing") 477 | # time.sleep(2) 478 | # 479 | # sdc.initialize() 480 | # 481 | # publish.single("core/edge/" + SDC_id + "/init_for_testing", 0, hostname=MQTT_HOST, 482 | # port=MQTT_PORT) 483 | # 484 | # # Creating threads 485 | # t2 = threading.Thread(target=consume_data_scenario2) 486 | # # Starting threads 487 | # t2.start() 488 | # # Wait until threads are completely executed 489 | # t2.join() 490 | # print("Test 2 is done!") 491 | # 492 | # publish.single("core/edge/" + SDC_id + "/done_to_test", "done", hostname=MQTT_HOST, port=MQTT_PORT) 493 | # time.sleep(3) 494 | # message_client.loop_stop() 495 | # message_client.disconnect() 496 | # time.sleep(2) 497 | # 498 | # message_client.connect(MQTT_HOST, MQTT_PORT, 60) 499 | # message_client.loop_start() 500 | # 501 | # # ---------- Scenario 3 502 | # 503 | # print("Truncate cache") 504 | # directory = os.path.join(".", "cache") 505 | # files = sorted(os.listdir(directory)) 506 | # for file_name in files: 507 | # full_path = os.path.join(directory, file_name) 508 | # os.remove(full_path) 509 | # 510 | # print("Start testing") 511 | # time.sleep(5) 512 | # 513 | # sdc.initialize() 514 | # 515 | # publish.single("core/edge/" + SDC_id + "/init_for_testing", 0, hostname=MQTT_HOST, 516 | # port=MQTT_PORT) 517 | # 518 | # # Creating threads 519 | # t3 = threading.Thread(target=consume_data_scenario3) 520 | # # Starting threads 521 | # t3.start() 522 | # # Wait until threads are completely executed 523 | # t3.join() 524 | # print("Test 3 is done!") 525 | # 526 | # publish.single("core/edge/" + SDC_id + "/done_to_test", "done", hostname=MQTT_HOST, port=MQTT_PORT) 527 | # time.sleep(3) 528 | # message_client.loop_stop() 529 | # message_client.disconnect() 530 | # time.sleep(2) 531 | # 532 | # message_client.connect(MQTT_HOST, MQTT_PORT, 60) 533 | # message_client.loop_start() 534 | # 535 | # # ---------- Scenario 4 536 | # 537 | # sdc.initialize() 538 | # 539 | # print("Truncate cache") 540 | # directory = os.path.join(".", "cache") 541 | # files = sorted(os.listdir(directory)) 542 | # for file_name in files: 543 | # full_path = os.path.join(directory, file_name) 544 | # os.remove(full_path) 545 | # 546 | # print("Start testing") 547 | # time.sleep(2) 548 | # 549 | # publish.single("core/edge/" + SDC_id + "/init_for_testing", 0, hostname=MQTT_HOST, 550 | # port=MQTT_PORT) 551 | # 552 | # # Creating threads 553 | # t4 = threading.Thread(target=consume_data_scenario4) 554 | # # Starting threads 555 | # t4.start() 556 | # # Wait until threads are completely executed 557 | # t4.join() 558 | # print("Test 4 is done!") 559 | # 560 | # publish.single("core/edge/" + SDC_id + "/done_to_test", "done", hostname=MQTT_HOST, port=MQTT_PORT) 561 | # time.sleep(3) 562 | # message_client.loop_stop() 563 | # message_client.disconnect() 564 | # time.sleep(2) 565 | 566 | loop_counter += 1 567 | 568 | scenario_counter += 1 569 | 570 | print("last") 571 | message_client.connect(MQTT_HOST, MQTT_PORT, 60) 572 | message_client.loop_start() 573 | 574 | publish.single("core/edge/" + SDC_id + "/all_test_complete", "complete", hostname=MQTT_HOST, port=MQTT_PORT, qos=2) 575 | print("All threads is done!") 576 | print("Notify - All test complete") 577 | time.sleep(2) 578 | 579 | # ## communication test section - start 580 | # while True: 581 | # time.sleep(1) 582 | # publish.single("core/edge/" + SDC_id + "/error", "5", hostname=MQTT_HOST, port=MQTT_PORT) 583 | 584 | # ## communication test section - end 585 | 586 | # publish.single("core/edge/" + SDC_id + "/error", "5", hostname=MQTT_HOST, port=MQTT_PORT) 587 | 588 | # development plan 589 | # 1. initialize cache (path, cache capacity, data queue) 590 | # 2. recv data 591 | # 3. calculate error 592 | # 4. send error 593 | # 5. RESTful API in this cache 594 | 595 | message_client.loop_stop() 596 | message_local_client.loop_stop() 597 | # Threads completely executed 598 | 599 | 600 | if __name__ == '__main__': 601 | try: 602 | main() 603 | except Exception as e: 604 | print("Exception: %s" % e) 605 | --------------------------------------------------------------------------------