├── .gitattributes ├── .gitignore ├── License.txt ├── command.py ├── README.md ├── mlogger.py └── mqtt-data-logger.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # This is the repository-wide ignore file. Suffixes below are ignored 2 | # i.e. committing not allowed. 3 | 4 | 5 | #The ZODB Data files 6 | data 7 | 8 | Data.fs 9 | Data.fs.lock 10 | Data.fs.tmp 11 | uwsgi.log 12 | uwsgi.pid 13 | 14 | # buildout junk 15 | .installed.cfg 16 | .mr.developer.cfg 17 | bin 18 | cache 19 | checkouts 20 | deploy.ini 21 | develop-eggs 22 | include 23 | lib 24 | local 25 | parts 26 | 27 | # archives 28 | *.tar 29 | *.tar.gz 30 | *.tar.bz2 31 | *.zip 32 | 33 | # editors 34 | *.swp 35 | *.elc 36 | *.el 37 | *~ 38 | *\#*\# 39 | 40 | # python 41 | *.pyc 42 | *.pyo 43 | *.bdf 44 | *.bak 45 | build/ 46 | dist/ 47 | 48 | # miscellaneous 49 | *.orig 50 | *.ori 51 | *.old 52 | *.svn 53 | *.rej 54 | *.egg-info 55 | 56 | # project specific 57 | config.json 58 | eggs/ 59 | jwt.key 60 | pip-selfcheck.json 61 | share/ 62 | -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | MIT LICENSE 2 | 3 | Copyright 2018 Steve Cope 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /command.py: -------------------------------------------------------------------------------- 1 | #!python3 2 | ###demo code provided by Steve Cope at www.steves-internet-guide.com 3 | ##email steve@steves-internet-guide.com 4 | ###Free to use for any purpose 5 | import sys, getopt 6 | options=dict() 7 | 8 | ##EDIT HERE ############### 9 | options["username"]="" 10 | options["password"]="" 11 | options["broker"]="127.0.0.1" 12 | options["port"]=1883 13 | options["verbose"]=True 14 | options["cname"]="" 15 | options["topics"]=[("",0)] 16 | options["storechangesonly"]=True 17 | options["JSON"]=False 18 | options["keepalive"]=60 19 | options["loglevel"]="WARNING" 20 | options["log_dir"]="mlogs" 21 | options["log_records"]=5000 22 | options["number_logs"]=0 23 | 24 | 25 | def command_input(options={}): 26 | topics_in=[] 27 | qos_in=[] 28 | 29 | valid_options=" --help -h or -b -p -t -q QOS -v -h \ 30 | -d logging debug -n Client ID or Name -u Username -P Password -s \ 31 | -l -r \ 32 | -f " 33 | print_options_flag=False 34 | try: 35 | opts, args = getopt.getopt(sys.argv[1:],"h:b:sdk:p:t:q:l:vn:u:P:l:r:f:j") 36 | except getopt.GetoptError: 37 | print (sys.argv[0],valid_options) 38 | sys.exit(2) 39 | qos=0 40 | 41 | for opt, arg in opts: 42 | if opt == '-h': 43 | options["broker"] = str(arg) 44 | elif opt == "-b": 45 | options["broker"] = str(arg) 46 | elif opt == "-k": 47 | options["keepalive"] = int(arg) 48 | elif opt =="-p": 49 | options["port"] = int(arg) 50 | elif opt =="-t": 51 | topics_in.append(arg) 52 | elif opt =="-q": 53 | qos_in.append(int(arg)) 54 | elif opt =="-n": 55 | options["cname"]=arg 56 | elif opt =="-d": 57 | options["loglevel"]="DEBUG" 58 | elif opt == "-P": 59 | options["password"] = str(arg) 60 | elif opt == "-u": 61 | options["username"] = str(arg) 62 | elif opt =="-v": 63 | options["verbose"]=True 64 | elif opt =="-s": 65 | options["storechangesonly"]=False 66 | elif opt =="-l": 67 | options["log_dir"]=str(arg) 68 | elif opt =="-r": 69 | options["log_records"]=int(arg) 70 | elif opt =="-f": 71 | options["number_logs"]=int(arg) 72 | elif opt =="-j": 73 | options["JSON"]=True 74 | 75 | lqos=len(qos_in) 76 | for i in range(len(topics_in)): 77 | if lqos >i: 78 | topics_in[i]=(topics_in[i],int(qos_in[i])) 79 | else: 80 | topics_in[i]=(topics_in[i],0) 81 | 82 | if topics_in: 83 | options["topics"]=topics_in #array with qos 84 | return options 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simple Python MQTT Data Logger 2 | 3 | This software uses the Python logger to create a logfile 4 | for all messages for all topics to which this MQTT client 5 | has subscribed. 6 | 7 | Note: by default it will only log changed messages. This is for sensors 8 | that send out their state a regular intervals but that state doesn't change 9 | The program is run from the command line 10 | 11 | You can subscribe to multiple topics. 12 | 13 | 14 | You need to provide the script with: 15 | 16 | * List of topics to monitor 17 | * broker name and port 18 | * username and password if needed. 19 | * base log directory and number of logs have defaults 20 | 21 | 22 | Valid command line Options: 23 | 24 | --help 25 | -h 26 | -b 27 | -p 28 | -t 29 | -q 30 | -v 31 | -d logging debug 32 | -n 33 | -u Username 34 | -P Password 35 | -s \ 36 | -l 37 | -r \ 38 | -f 40 | 41 | 42 | # Install dependencies 43 | 44 | You need the paho mqtt library. Using pip3: 45 | 46 | pip install paho-mqtt 47 | 48 | # Example Usage: 49 | 50 | You will always need to specify the broker name or IP address 51 | and the topics to log. 52 | 53 | Note: you may not need to use the python prefix or may 54 | need to use python3 mqtt_data_logger.py (Linux) 55 | 56 | Specify broker and topics 57 | 58 | python mqtt_data_logger.py -b 192.168.1.157 -t sensors/# 59 | 60 | Specify broker and multiple topics 61 | 62 | python mqtt_data_logger.py -b 192.168.1.157 -t sensors/# -t home/# 63 | 64 | 65 | Log All Data (plain text format): 66 | 67 | python mqtt_data_logger.py b 192.168.1.157 -t sensors/# -s 68 | 69 | Log All Data (JSON format): 70 | 71 | python mqtt_data_logger.py b 192.168.1.157 -t sensors/# -s -j 72 | 73 | Specify the client name used by the logger 74 | 75 | python mqtt_data_logger.py b 192.168.1.157 -t sensors/# -n data-logger 76 | 77 | Specify the log directory 78 | 79 | python mqtt_data_logger.py b 192.168.1.157 -t sensors/# -l mylogs 80 | 81 | # Logger Class 82 | 83 | The class is implemented in a module called m_logger.py (message logger). 84 | 85 | To create an instance you need to supply three parameters: 86 | 87 | * The log directory- defaults to mlogs 88 | * Number of records to log per log- defaults to 5000 89 | * Number of logs. 0 for no limit.- defaults to 0 90 | 91 | log=m_logger(log_dir="logs",log_recs=5000,number_logs=0): 92 | 93 | The logger creates the log files in the directory using the current date and time for the directory names. 94 | 95 | The format is month-day-hour-minute e.g. 96 | 97 | You can log data either in plain text format or JSON format. 98 | 99 | To log data either in plain text then use the 100 | 101 | log_data(data) method. 102 | 103 | To log data as JSON encoded data call the 104 | 105 | log_json(data) method. 106 | 107 | Both method takes a single parameter containing the data to log as a string, list or dictionary.. 108 | 109 | e.g. 110 | 111 | log.log_data(data) 112 | 113 | or 114 | 115 | log.log_json(data) 116 | 117 | The log file will contain the data as plain text or JSON encoded data strings each on a newline. 118 | 119 | The logger will return True if successful and False if not. 120 | 121 | To prevent loss of data in the case of computer failure the logs are continuously flushed to disk . 122 | 123 | Read more about this application here: 124 | 125 | http://www.steves-internet-guide.com/simple-python-mqtt-data-logger/ 126 | -------------------------------------------------------------------------------- /mlogger.py: -------------------------------------------------------------------------------- 1 | ###demo code provided by Steve Cope at www.steves-internet-guide.com 2 | ##email steve@steves-internet-guide.com 3 | ###Free to use for any purpose 4 | """ 5 | implements data logging class 6 | """ 7 | import time,os,json,logging 8 | 9 | ############### 10 | class m_logger(object): 11 | """Class for logging data to a file. You can set the maximim bunber 12 | of messages in a file the default is 5000. Setting number_logs to 0 13 | means that there is no limit on the number of logs.When the file is full 14 | a new file is created.Log files are stored under a root directory 15 | and a sub directory that uses the timestamp for the directory name 16 | Log file data is flushed immediately to disk so that data is not lost. 17 | Data can be stored as plain text or in JSON format """ 18 | def __init__(self,log_dir="mlogs",log_recs=5000,number_logs=0): 19 | self.log_dir=log_dir 20 | self.log_recs=log_recs 21 | self.number_logs=number_logs 22 | self.count=0 23 | self.log_dir=self.create_log_dir(self.log_dir) 24 | self.fo=self.get_log_name(self.log_dir,self.count) 25 | self.new_file_flag=0 26 | self.writecount=0 27 | self.timenow=time.time() 28 | self.flush_flag=True 29 | self.flush_time=2 #flush logs to disk every 2 seconds 30 | def __flushlogs(self): # write to disk 31 | self.fo.flush() 32 | #logging.info("flushing logs") 33 | os.fsync(self.fo.fileno()) 34 | self.timenow=time.time() 35 | def __del__(self): 36 | if not self.fo.closed: 37 | print("closing log file") 38 | self.fo.close() 39 | def close_file(self): 40 | if not self.fo.closed: 41 | print("closing log file") 42 | self.fo.close() 43 | def create_log_dir(self,log_dir): 44 | """ 45 | Function for creating new log directories 46 | using the timestamp for the name 47 | """ 48 | self.t=time.localtime(time.time()) 49 | self.time_stamp=(str(self.t[0])+"-"+str(self.t[1])+"-"+str(self.t[2])+"-"+\ 50 | str(self.t[3])+"-"+str(self.t[4])) 51 | logging.info("creating sub directory"+str(self.time_stamp)) 52 | try: 53 | os.stat(self.log_dir) 54 | except: 55 | os.mkdir(self.log_dir) 56 | self.log_sub_dir=self.log_dir+"/"+self.time_stamp 57 | try: 58 | os.stat(self.log_sub_dir) 59 | except: 60 | os.mkdir(self.log_sub_dir) 61 | return(self.log_sub_dir) 62 | 63 | def get_log_name(self,log_dir,count): 64 | """get log files and directories""" 65 | self.log_numbr="{0:003d}".format(count) 66 | logging.info("s is"+str(self.log_numbr)) 67 | self.file_name=self.log_dir+"/"+"log"+self.log_numbr 68 | logging.info("creating log file "+self.file_name) 69 | f = open(self.file_name,'w') #clears file if it exists 70 | f.close() 71 | f=open(self.file_name, 'a') 72 | return(f) 73 | def log_json(self,data): #data is a JavaScript object 74 | jdata=json.dumps(data)+"\n" 75 | self.log_data(jdata) 76 | 77 | def log_data(self, data): 78 | self.data=data 79 | try: 80 | self.fo.write(data) 81 | self.writecount+=1 82 | self.__flushlogs() 83 | if self.writecount>=self.log_recs: 84 | self.count+=1 #counts number of logs 85 | if self.count>self.number_logs and self.number_logs !=0 : 86 | logging.info("too many logs: starting from 0") 87 | self.count=0 #reset 88 | self.fo=self.get_log_name(self.log_dir,self.count) 89 | self.writecount=0 90 | except BaseException as e: 91 | logging.error("Error on_data: %s" % str(e)) 92 | return False 93 | return True 94 | 95 | -------------------------------------------------------------------------------- /mqtt-data-logger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | #If Running in Windows use top line and edit according to your python 3 | #location and version. If running on Linux delete the top line. 4 | ###demo code provided by Steve Cope at www.steves-internet-guide.com 5 | ##email steve@steves-internet-guide.com 6 | ###Free to use for any purpose 7 | """ 8 | This will log messages to file.Los time,message and topic as JSON data 9 | """ 10 | #updated 28-oct-2018 11 | mqttclient_log=False #MQTT client logs showing messages 12 | Log_worker_flag=True 13 | import paho.mqtt.client as mqtt 14 | import json 15 | import os 16 | import time 17 | import sys, getopt,random 18 | import logging 19 | import mlogger as mlogger 20 | import threading 21 | from queue import Queue 22 | from command import command_input 23 | import command 24 | import sys 25 | print("Python version is", sys.version_info) 26 | 27 | q=Queue() 28 | 29 | ##helper functions 30 | def convert(t): 31 | d="" 32 | for c in t: # replace all chars outside BMP with a ! 33 | d =d+(c if ord(c) < 0x10000 else '!') 34 | return(d) 35 | ### 36 | 37 | class MQTTClient(mqtt.Client):#extend the paho client class 38 | run_flag=False #global flag used in multi loop 39 | def __init__(self,cname,**kwargs): 40 | super(MQTTClient, self).__init__(cname,**kwargs) 41 | self.last_pub_time=time.time() 42 | self.topic_ack=[] #used to track subscribed topics 43 | self.run_flag=True 44 | self.submitted_flag=False #used for connections 45 | self.subscribe_flag=False 46 | self.bad_connection_flag=False 47 | self.bad_count=0 48 | self.connected_flag=False 49 | self.connect_flag=False #used in multi loop 50 | self.disconnect_flag=False 51 | self.disconnect_time=0.0 52 | self.pub_msg_count=0 53 | self.pub_flag=False 54 | self.sub_topic="" 55 | self.sub_topics=[] #multiple topics 56 | self.sub_qos=0 57 | self.devices=[] 58 | self.broker="" 59 | self.port=1883 60 | self.keepalive=60 61 | self.run_forever=False 62 | self.cname="" 63 | self.delay=10 #retry interval 64 | self.retry_time=time.time() 65 | 66 | def Initialise_clients(cname,mqttclient_log=False,cleansession=True,flags=""): 67 | #flags set 68 | print("initialising clients") 69 | logging.info("initialising clients") 70 | client= MQTTClient(cname,clean_session=cleansession) 71 | client.cname=cname 72 | client.on_connect= on_connect #attach function to callback 73 | client.on_message=on_message #attach function to callback 74 | #client.on_disconnect=on_disconnect 75 | #client.on_subscribe=on_subscribe 76 | if mqttclient_log: 77 | client.on_log=on_log 78 | return client 79 | 80 | def on_connect(client, userdata, flags, rc): 81 | """ 82 | set the bad connection flag for rc >0, Sets onnected_flag if connected ok 83 | also subscribes to topics 84 | """ 85 | logging.debug("Connected flags"+str(flags)+"result code "\ 86 | +str(rc)+"client1_id") 87 | if rc==0: 88 | 89 | client.connected_flag=True #old clients use this 90 | client.bad_connection_flag=False 91 | if client.sub_topic!="": #single topic 92 | logging.debug("subscribing "+str(client.sub_topic)) 93 | print("subscribing in on_connect") 94 | topic=client.sub_topic 95 | if client.sub_qos!=0: 96 | qos=client.sub_qos 97 | client.subscribe(topic,qos) 98 | elif client.sub_topics!="": 99 | #print("subscribing in on_connect multiple") 100 | client.subscribe(client.sub_topics) 101 | 102 | else: 103 | print("set bad connection flag") 104 | client.bad_connection_flag=True # 105 | client.bad_count +=1 106 | client.connected_flag=False # 107 | def on_message(client,userdata, msg): 108 | topic=msg.topic 109 | m_decode=str(msg.payload.decode("utf-8","ignore")) 110 | message_handler(client,m_decode,topic) 111 | #print("message received") 112 | 113 | def message_handler(client,msg,topic): 114 | 115 | tnow=time.localtime(time.time()) 116 | 117 | if options["JSON"]: 118 | data=dict() 119 | try: 120 | msg=json.loads(msg)#convert to Javascript before saving 121 | #print("json data") 122 | except: 123 | pass 124 | #print("not already json") 125 | data["time"]=tnow 126 | data["topic"]=topic 127 | data["message"]=msg 128 | #print("Logging JSON format") 129 | else: 130 | data=time.asctime(tnow)+" "+topic+" "+msg+"\n" 131 | #print("Logging plain text") 132 | 133 | if command.options["storechangesonly"]: 134 | if has_changed(client,topic,msg): 135 | client.q.put(data) #put messages on queue 136 | else: 137 | client.q.put(data) #put messages on queue 138 | 139 | def has_changed(client,topic,msg): 140 | topic2=topic.lower() 141 | if topic2.find("control")!=-1: 142 | return False 143 | if topic in client.last_message: 144 | if client.last_message[topic]==msg: 145 | return False 146 | client.last_message[topic]=msg 147 | return True 148 | ### 149 | def log_worker(): 150 | """runs in own thread to log data from queue""" 151 | while Log_worker_flag: 152 | time.sleep(0.01) 153 | while not q.empty(): 154 | results = q.get() 155 | if results is None: 156 | continue 157 | if options["JSON"]: 158 | log.log_json(results) 159 | else: 160 | log.log_data(results) 161 | #print("message saved ",results["message"]) 162 | log.close_file() 163 | 164 | # MAIN PROGRAM 165 | options=command.options 166 | 167 | if __name__ == "__main__" and len(sys.argv)>=2: 168 | options=command_input(options) 169 | else: 170 | print("Need broker name and topics to continue.. exiting") 171 | raise SystemExit(1) 172 | 173 | 174 | if not options["cname"]: 175 | r=random.randrange(1,10000) 176 | cname="logger-"+str(r) 177 | else: 178 | cname="logger-"+str(options["cname"]) 179 | log_dir=options["log_dir"] 180 | log_records=options["log_records"] 181 | number_logs=options["number_logs"] 182 | log=mlogger.m_logger(log_dir,log_records,number_logs) #create log object 183 | print("Log Directory =",log_dir) 184 | print("Log records per log =",log_records) 185 | if number_logs==0: 186 | print("Max logs = Unlimited") 187 | else: 188 | print("Max logs =",number_logs) 189 | 190 | 191 | logging.info("creating client"+cname) 192 | 193 | client=Initialise_clients(cname,mqttclient_log,False)#create and initialise client object 194 | if options["username"] !="": 195 | client.username_pw_set(options["username"], options["password"]) 196 | 197 | client.sub_topics=options["topics"] 198 | client.broker=options["broker"] 199 | client.port=options["port"] 200 | 201 | if options["JSON"]: 202 | print("Logging JSON format") 203 | else: 204 | print("Logging plain text") 205 | if options["storechangesonly"]: 206 | print("starting storing only changed data") 207 | else: 208 | print("starting storing all data") 209 | 210 | ## 211 | #Log_worker_flag=True 212 | t = threading.Thread(target=log_worker) #start logger 213 | t.start() #start logging thread 214 | ### 215 | 216 | client.last_message=dict() 217 | client.q=q #make queue available as part of client 218 | 219 | 220 | 221 | 222 | try: 223 | res=client.connect(client.broker,client.port) #connect to broker 224 | client.loop_start() #start loop 225 | 226 | except: 227 | logging.debug("connection to ",client.broker," failed") 228 | raise SystemExit("connection failed") 229 | try: 230 | while True: 231 | time.sleep(1) 232 | pass 233 | 234 | except KeyboardInterrupt: 235 | print("interrrupted by keyboard") 236 | 237 | client.loop_stop() #start loop 238 | Log_worker_flag=False #stop logging thread 239 | time.sleep(5) 240 | 241 | --------------------------------------------------------------------------------