├── .gitattributes
├── .gitignore
├── License.txt
├── README.md
├── command.py
├── mlogger.py
├── mqtt-data-logger.py
└── mqttdatalogger.py


/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # This is the repository-wide ignore file. Suffixes below are ignored
 2 | # i.e. committing not allowed.
 3 | 
 4 | 
 5 | #The ZODB Data files
 6 | data
 7 | 
 8 | Data.fs
 9 | Data.fs.lock
10 | Data.fs.tmp
11 | uwsgi.log
12 | uwsgi.pid
13 | 
14 | # buildout junk
15 | .installed.cfg
16 | .mr.developer.cfg
17 | bin
18 | cache
19 | checkouts
20 | deploy.ini
21 | develop-eggs
22 | include
23 | lib
24 | local
25 | parts
26 | 
27 | # archives
28 | *.tar
29 | *.tar.gz
30 | *.tar.bz2
31 | *.zip
32 | 
33 | # editors
34 | *.swp
35 | *.elc
36 | *.el
37 | *~
38 | *\#*\#
39 | 
40 | # python
41 | *.pyc
42 | *.pyo
43 | *.bdf
44 | *.bak
45 | build/
46 | dist/
47 | 
48 | # miscellaneous
49 | *.orig
50 | *.ori
51 | *.old
52 | *.svn
53 | *.rej
54 | *.egg-info
55 | 
56 | # project specific
57 | config.json
58 | eggs/
59 | jwt.key
60 | pip-selfcheck.json
61 | share/
62 | 


--------------------------------------------------------------------------------
/License.txt:
--------------------------------------------------------------------------------
 1 | MIT LICENSE
 2 | 
 3 | Copyright 2018 Steve Cope
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 6 | 
 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 8 | 
 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
10 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Simple Python MQTT Data Logger
  2 | 
  3 | This software uses the Python logger to create a logfile
  4 | for all messages for all topics to which this MQTT client
  5 | has subscribed.
  6 | Note: by default it will only log changed messages. This is for sensors 
  7 | that send out their state a regular intervals but that state doesn't change
  8 | The program is run from the command line
  9 | You can subscribe to multiple topics.
 10 | 
 11 | 
 12 | 
 13 | 
 14 | You need to provide the script with:
 15 | 
 16 |     List of topics to monitor
 17 |     broker name and port
 18 |     username and password if needed.
 19 |     base log directory and number of logs have defaults
 20 | Valid command line Options:
 21 | --help <help>
 22 | -h <broker> 
 23 | -b <broker> 
 24 | -p <port>
 25 | -t <topic> 
 26 | -q <QOS>
 27 | -v <verbose>
 28 | -d logging debug 
 29 | -n <Client ID or Name>
 30 | -u Username 
 31 | -P Password
 32 | -s <store all data>\
 33 | -l <log directory default= mlogs> 
 34 | -r <number of records default=100>\
 35 | -f <number of log files default= unlimited"
 36 | -c <log csv format>
 37 | -j <log json format>
 38 | 
 39 | The logger will log data as JSON or csv using the -j or -c flags.
 40 | Incoming data can be plain text or json formatted data.
 41 | currnetly the json data cannot contain nested elements.
 42 | When storing logs as csv the headers are automatically extraced from the json keys is the data is json formated.
 43 | If the incoming data is in csv format it will be logged as plain text.
 44 | 	
 45 | 	Example Usage:
 46 | 
 47 | You will always need to specify the broker name or IP address 
 48 | and the topics to log
 49 | 
 50 | Note: you may not need to use the python prefix or may 
 51 | need to use python3 mqtt-data-logger.py (Linux)
 52 | 
 53 | Specify broker and topics 
 54 | 
 55 |     python mqtt-data-logger.py -b 192.168.1.157 -t sensors/#
 56 | 
 57 | Specify broker and multiple topics
 58 | 
 59 |     python mqtt-data-logger.py -b 192.168.1.157 -t sensors/# -t  home/#
 60 | 
 61 | Log as Json
 62 | 
 63 |     python mqtt-data-logger.py -b 192.168.1.157 -t sensors/# -j
 64 | 	
 65 | Log as CSV
 66 | 
 67 |     python mqtt-data-logger.py -b 192.168.1.157 -t sensors/# -c
 68 | 
 69 | Log All Data:
 70 | 
 71 |     python mqtt-data-logger.py b 192.168.1.157 -t sensors/# -s 
 72 | 
 73 | Specify the client name used by the logger
 74 | 
 75 |     python mqtt-data-logger.py b 192.168.1.157 -t sensors/# -n data-logger
 76 | 
 77 | Specify the log directory
 78 | 
 79 |     python mqtt-data-logger.py b 192.168.1.157 -t sensors/# -l mylogs
 80 |  
 81 | ---------
 82 | Logger Class
 83 | 
 84 | The class is implemented in a module called m_logger.py (message logger).
 85 | 
 86 | To create an instance you need to supply three parameters:
 87 | 
 88 |     The log directory- defaults to mlogs
 89 |     Number of records to log per log- defaults to 5000
 90 |     Number of logs. 0 for no limit.- defaults to 0
 91 | 
 92 | log=m_logger(log_dir="logs",log_recs=5000,number_logs=0):
 93 | 
 94 | The logger creates the log files in the directory using the current date and time for the directory names.
 95 | 
 96 | The format is month-day-hour-minute e.g.
 97 | 
 98 | 
 99 | You can log data either in plain text format or JSON format.
100 | 
101 | To log data either in plain text then use the
102 | 
103 |     log_data(data) method.
104 | 
105 | To log data as JSON encoded data call the
106 | 
107 |     log_json(data) method.
108 | 
109 | Both method takes a single parameter containing the data to log as a string, list or dictionary..
110 | 
111 | e.g.
112 | 
113 | log.log_data(data) 
114 | or
115 | log.log_json(data)
116 | 
117 | #The log file will contain the data as 
118 | #plain text or  JSON encoded data strings or csv
119 | #each on a newline.
120 | 
121 | The logger will return True if successful and False if not.
122 | 
123 | To prevent loss of data in the case of computer failure the logs are continuously flushed to disk .
124 |  
125 |  Read more about this application here:
126 | http://www.steves-internet-guide.com/simple-python-mqtt-data-logger/
127 | 


--------------------------------------------------------------------------------
/command.py:
--------------------------------------------------------------------------------
 1 | #!python3
 2 | ###demo code provided by Steve Cope at www.steves-internet-guide.com
 3 | ##email steve@steves-internet-guide.com
 4 | ###Free to use for any purpose
 5 | import sys, getopt
 6 | options=dict()
 7 | 
 8 | ##EDIT HERE ###############
 9 | options["username"]=""
10 | options["password"]=""
11 | options["broker"]="192.168.1.37"
12 | options["port"]=1883
13 | options["verbose"]=True
14 | options["cname"]=""
15 | options["topics"]=[("#",0)]
16 | options["storechangesonly"]=True
17 | options["keepalive"]=60
18 | options["loglevel"]="WARNING"
19 | options["log_dir"]="mlogs"
20 | options["log_records"]=5000
21 | options["number_logs"]=0
22 | options["JSON"]=False
23 | options["CSV"]=False
24 | options["header_file"]="data.csv"
25 | options["fname"]="data.csv"
26 | options["header_flag"]=False
27 | 
28 | 
29 | def command_input(options={}):
30 |     topics_in=[]
31 |     qos_in=[]
32 | 
33 |     valid_options=" --help <help> -h or -b <broker> -p <port>-t <topic> -q QOS -v <verbose> -h <help>\
34 |  -d logging debug  -n Client ID or Name -u Username -P Password -s <store all data>\
35 | -l <log directory default= mlogs> -r <number of records default=100>\
36 | -f <number of log files default= unlimited> -j <log in json format> -c <log in csvformat>"
37 |     print_options_flag=False
38 |     try:
39 |       opts, args = getopt.getopt(sys.argv[1:],"h:b:cjsdk:p:t:q:l:vn:u:P:l:r:f:")
40 |     except getopt.GetoptError:
41 |       print (sys.argv[0],valid_options)
42 |       sys.exit(2)
43 |     qos=0
44 | 
45 |     for opt, arg in opts:
46 |         if opt == '-h':
47 |              options["broker"] = str(arg)
48 |         elif opt == "-b":
49 |              options["broker"] = str(arg)
50 |         elif opt == "-k":
51 |              options["keepalive"] = int(arg)
52 |         elif opt =="-p":
53 |             options["port"] = int(arg)
54 |         elif opt =="-t":
55 |             topics_in.append(arg)
56 |         elif opt =="-q":
57 |              qos_in.append(int(arg))
58 |         elif opt =="-n":
59 |              options["cname"]=arg
60 |         elif opt =="-d":
61 |             options["loglevel"]="DEBUG"
62 |         elif opt == "-P":
63 |              options["password"] = str(arg)
64 |         elif opt == "-u":
65 |              options["username"] = str(arg)
66 |         elif opt =="-v":
67 |             options["verbose"]=True
68 |         elif opt =="-s":
69 |             options["storechangesonly"]=False
70 |         elif opt =="-l":
71 |             options["log_dir"]=str(arg)
72 |         elif opt =="-r":
73 |             options["log_records"]=int(arg)
74 |         elif opt =="-f":
75 |             options["number_logs"]=int(arg)
76 |         elif opt =="-j":
77 |             options["JSON"]=True
78 |         elif opt =="-c":
79 |             options["CSV"]=True
80 | 
81 | 
82 |     lqos=len(qos_in)
83 |     for i in range(len(topics_in)):
84 |         if lqos >i:
85 |             topics_in[i]=(topics_in[i],int(qos_in[i]))
86 |         else:
87 |             topics_in[i]=(topics_in[i],0)
88 | 
89 |     if topics_in:
90 |         options["topics"]=topics_in #array with qos
91 |     return options
92 | 


--------------------------------------------------------------------------------
/mlogger.py:
--------------------------------------------------------------------------------
  1 | ###demo code provided by Steve Cope at www.steves-internet-guide.com
  2 | ##email steve@steves-internet-guide.com
  3 | ###Free to use for any purpose
  4 | """
  5 | implements data logging class
  6 | """
  7 | import time,os,json,logging
  8 | 
  9 | ###############
 10 | class m_logger(object):
 11 |     """Class for logging data to a file. You can set the maximim number
 12 | of messages in a file the default is 5000. Setting number_logs to 0
 13 | means that there is no limit on the number of logs.When the file is full
 14 | a new file is created.Log files are stored under a root directory
 15 | and a sub directory that uses the timestamp for the directory name
 16 | Log file data is flushed immediately to disk so that data is not lost.
 17 | Data can be stored as plain text or in JSON format """
 18 |     def __init__(self,log_dir="mlogs",log_recs=5000,number_logs=0,csv_flag=False):
 19 |         self.log_dir=log_dir
 20 |         self.log_recs=log_recs
 21 |         self.number_logs=number_logs
 22 |         self.count=0
 23 |         self.log_dir=self.create_log_dir(self.log_dir)
 24 |         self.fo=self.get_log_name(self.log_dir,self.count)
 25 |         self.new_file_flag=0
 26 |         self.writecount=0
 27 |         self.timenow=time.time()
 28 |         self.flush_flag=True
 29 |         self.flush_time=2 #flush logs to disk every 2 seconds
 30 |         self.csv_flag = csv_flag
 31 |         self.columns=""
 32 | 
 33 |     def extract_columns(self,data):
 34 |         columns=""
 35 | 
 36 |         #data=flatten_dict(msg)
 37 |         for key in data:
 38 |             #print("key =",key)
 39 |             if columns =="":
 40 |                 columns=key
 41 |             else:
 42 |                 columns=columns+","+key
 43 |         #print(columns)
 44 |         return(columns)
 45 |     def extract_data(self,data):
 46 |         line_out=""
 47 |         for key in data:
 48 |             #print("here ",data[key])
 49 |             if line_out =="":
 50 |                 line_out=str(data[key])
 51 |             else:
 52 |                 line_out=line_out+","+str(data[key])
 53 |         #print(line_out)
 54 |         return(line_out)
 55 |     def __flushlogs(self): # write to disk 
 56 |         self.fo.flush()
 57 |         #logging.info("flushing logs")
 58 |         os.fsync(self.fo.fileno())
 59 |         self.timenow=time.time()
 60 |     def __del__(self):
 61 |         if not self.fo.closed:
 62 |             print("closing log file")
 63 |             self.fo.close()
 64 |     def close_file(self):
 65 |         if not self.fo.closed:
 66 |             print("closing log file")
 67 |             self.fo.close()
 68 |     def create_log_dir(self,log_dir):
 69 |         """
 70 |         Function for creating new log directories
 71 |         using the timestamp for the name
 72 |         """   
 73 |         self.t=time.localtime(time.time())      
 74 |         self.time_stamp=(str(self.t[1])+"-"+str(self.t[2])+"-"+\
 75 |         str(self.t[3])+"-"+str(self.t[4]))                                                                             
 76 |         logging.info("creating sub directory"+str(self.time_stamp))
 77 |         try:
 78 |             os.stat(self.log_dir)
 79 |         except:
 80 |             os.mkdir(self.log_dir) 
 81 |         self.log_sub_dir=self.log_dir+"/"+self.time_stamp 
 82 |         try:
 83 |             os.stat(self.log_sub_dir)
 84 |         except:
 85 |             os.mkdir(self.log_sub_dir)
 86 |         return(self.log_sub_dir)
 87 | 
 88 |     def get_log_name(self,log_dir,count):
 89 |         """get log files and directories"""
 90 |         self.log_numbr="{0:003d}".format(count)
 91 |         logging.info("s is"+str(self.log_numbr))
 92 |         self.file_name=self.log_dir+"/"+"log"+self.log_numbr +".log"
 93 |         logging.info("creating log file "+self.file_name)
 94 |         f = open(self.file_name,'w') #clears file if it exists
 95 |         f.close()
 96 |         f=open(self.file_name, 'a')
 97 |         return(f)
 98 |     def log_json(self,data): #data is a JavaScript object
 99 |         jdata=json.dumps(data)+"\n"
100 |         self.log_data(jdata)                                                                                          
101 |     def log_csv(self,data): #data is a JavaScript object that needs converting
102 |         #print("logging csv ",self.writecount)
103 |         if self.writecount==0:
104 |             self.columns=self.extract_columns(data)+"\n"
105 |             csv_flag=True
106 |             self.log_data(self.columns)
107 |         else:
108 |             csv_data=self.extract_data(data)+"\n"
109 |             self.log_data(csv_data)
110 |     def log_data(self, data):
111 |         #print("logging data")
112 |         self.data=data
113 |         try:
114 |             self.fo.write(data)
115 |             self.writecount+=1
116 |             self.__flushlogs()
117 |             if self.writecount>=self.log_recs:
118 |                 self.count+=1 #counts number of logs
119 |                 if self.count>self.number_logs and self.number_logs !=0 :
120 |                       logging.info("too many logs: starting from 0")
121 |                       self.count=0 #reset
122 |                 self.fo=self.get_log_name(self.log_dir,self.count)
123 |                 self.writecount=0
124 |         except BaseException as e:
125 |             logging.error("Error on_data: %s" % str(e))
126 |             return False
127 |         return True
128 | 
129 | 


--------------------------------------------------------------------------------
/mqtt-data-logger.py:
--------------------------------------------------------------------------------
  1 | #!c:\python34\python.exe
  2 | #!/usr/bin/env python
  3 | #If Running in Windows use top line and edit according to your python
  4 | #location and version. If running on Linux delete the top line.
  5 | ###demo code provided by Steve Cope at www.steves-internet-guide.com
  6 | ##email steve@steves-internet-guide.com
  7 | ###Free to use for any purpose
  8 | """
  9 | This will log messages to file.Los time,message and topic as JSON data
 10 | """
 11 | #updated 18-oct-2023
 12 | mqttclient_log=False #MQTT client logs showing messages
 13 | Log_worker_flag=True
 14 | import paho.mqtt.client as mqtt
 15 | import json
 16 | import os
 17 | import time
 18 | import sys, getopt,random
 19 | import logging
 20 | import mlogger as mlogger
 21 | import threading
 22 | from queue import Queue
 23 | from command import command_input
 24 | import command
 25 | import sys
 26 | print("Python version is", sys.version_info)
 27 | 
 28 | q=Queue()
 29 | 
 30 | ##helper functions
 31 | def convert(t):
 32 |     d=""
 33 |     for c in t:  # replace all chars outside BMP with a !
 34 |             d =d+(c if ord(c) < 0x10000 else '!')
 35 |     return(d)
 36 | ###
 37 | 
 38 | class MQTTClient(mqtt.Client):#extend the paho client class
 39 |    run_flag=False #global flag used in multi loop
 40 |    def __init__(self,cname,**kwargs):
 41 |       super(MQTTClient, self).__init__(cname,**kwargs)
 42 |       self.last_pub_time=time.time()
 43 |       self.topic_ack=[] #used to track subscribed topics
 44 |       self.run_flag=True
 45 |       self.submitted_flag=False #used for connections
 46 |       self.subscribe_flag=False
 47 |       self.bad_connection_flag=False
 48 |       self.bad_count=0
 49 |       self.connected_flag=False
 50 |       self.connect_flag=False #used in multi loop
 51 |       self.disconnect_flag=False
 52 |       self.disconnect_time=0.0
 53 |       self.pub_msg_count=0
 54 |       self.pub_flag=False
 55 |       self.sub_topic=""
 56 |       #self.sub_topics=["tele/#","stat/#"] #multiple topics
 57 |       self.sub_qos=0
 58 |       self.devices=[]
 59 |       self.broker="mqtt2.home"
 60 |       self.port=1883
 61 |       self.keepalive=60
 62 |       self.run_forever=False
 63 |       self.cname=""
 64 |       self.delay=10 #retry interval
 65 |       self.retry_time=time.time()
 66 | 
 67 | def Initialise_clients(cname,mqttclient_log=False,cleansession=True,flags=""):
 68 |     #flags set
 69 |    print("initialising clients")
 70 |    logging.info("initialising clients")
 71 |    client= MQTTClient(cname,clean_session=cleansession)
 72 |    client.cname=cname
 73 |    client.on_connect= on_connect        #attach function to callback
 74 |    client.on_message=on_message        #attach function to callback
 75 |    #client.on_disconnect=on_disconnect
 76 |    #client.on_subscribe=on_subscribe
 77 |    if mqttclient_log:
 78 |       client.on_log=on_log
 79 |    return client
 80 | 
 81 | def on_connect(client, userdata, flags, rc):
 82 |    """
 83 |    set the bad connection flag for rc >0, Sets onnected_flag if connected ok
 84 |    also subscribes to topics
 85 |    """
 86 |    logging.debug("Connected flags"+str(flags)+"result code "\
 87 |     +str(rc)+"client1_id")
 88 |    if rc==0:
 89 |       
 90 |       client.connected_flag=True #old clients use this
 91 |       client.bad_connection_flag=False
 92 |       if client.sub_topic!="": #single topic
 93 |          print("subscribing "+str(client.sub_topic))
 94 |          print("subscribing in on_connect ")
 95 |          topic=client.sub_topics
 96 |          if client.sub_qos!=0:
 97 |             client.subscribe(topic,qos)
 98 |       elif client.sub_topics!="":
 99 |          print("subscribing in on_connect multiple",client.sub_topics)
100 |          client.subscribe(client.sub_topics)
101 | 
102 |    else:
103 |      print("set bad connection flag")
104 |      client.bad_connection_flag=True #
105 |      client.bad_count +=1
106 |      client.connected_flag=False #
107 | def on_message(client,userdata, msg):
108 |     topic=msg.topic
109 |     m_decode=str(msg.payload.decode("utf-8","ignore"))
110 |     message_handler(client,m_decode,topic)
111 |     #print("message received")
112 |     
113 | def message_handler(client,msg,topic):
114 |     data=dict()
115 |     tnow=time.ctime()
116 |     #m=time.asctime(tnow)+" "+topic+" "+msg
117 |     try:
118 |         msg=json.loads(msg)#convert to Javascript before saving
119 |         #print("json data")
120 |     except:
121 |         pass
122 |         #print("not already json")
123 | 
124 |     data["time"]=tnow
125 |     data["time_ms"]=time.time()
126 |     data["topic"]=topic
127 |     if csv_flag:
128 |         try:
129 |             keys=msg.keys()
130 |             for key in keys:
131 |                 data[key]=msg[key]
132 |         except:
133 |             data["message"]=msg 
134 |     else:             
135 |         data["message"]=msg
136 |     
137 |     #data["message"]=msg
138 |     if command.options["storechangesonly"]:
139 |         if has_changed(client,topic,msg):
140 |             client.q.put(data) #put messages on queue
141 |     else:
142 |         client.q.put(data) #put messages on queue
143 | 
144 | def has_changed(client,topic,msg):
145 |     topic2=topic.lower()
146 |     if topic2.find("control")!=-1:
147 |         return False
148 |     if topic in client.last_message:
149 |         if client.last_message[topic]==msg:
150 |             return False
151 |     client.last_message[topic]=msg
152 |     return True
153 | ###
154 | def log_worker():
155 |     """runs in own thread to log data from queue"""
156 |     while Log_worker_flag:
157 |         time.sleep(0.01)
158 |         while not q.empty():
159 |             results = q.get()
160 |             if results is None:
161 |                 continue
162 |  
163 |             if csv_flag:
164 |                 log.log_csv(results)
165 |             else:
166 |                 log.log_json(results)
167 |   
168 |             
169 |             #print("message saved ",results["message"])
170 |     log.close_file()
171 | 
172 | # MAIN PROGRAM
173 | options=command.options
174 | 
175 | if __name__ == "__main__" and len(sys.argv)>=2:
176 |     options=command_input(options)
177 | else:
178 |     print("Need broker name and topics to continue.. exiting")
179 |     #raise SystemExit(1)
180 | 
181 | 
182 | if not options["cname"]:
183 |     r=random.randrange(1,10000)
184 |     cname="logger-"+str(r)
185 | else:
186 |     cname="logger-"+str(options["cname"])
187 | log_dir=options["log_dir"]
188 | log_records=options["log_records"]
189 | number_logs=options["number_logs"]
190 | log=mlogger.m_logger(log_dir,log_records,number_logs) #create log object
191 | print("Log Directory =",log_dir)
192 | print("Log records per log =",log_records)
193 | if number_logs==0:
194 |     print("Max logs = Unlimited")
195 | else:
196 |     print("Max logs  =",number_logs)
197 |     
198 | 
199 | logging.info("creating client"+cname)
200 | 
201 | client=Initialise_clients(cname,mqttclient_log,False)#create and initialise client object
202 | if options["username"] !="":
203 |     client.username_pw_set(options["username"], options["password"])
204 | 
205 | client.sub_topics=options["topics"]
206 | client.broker=options["broker"]
207 | client.port=options["port"]
208 | json_flag=False
209 | csv_flag=False
210 | 
211 | if options["JSON"]:
212 |     print("Logging JSON format")
213 |     json_flag=True
214 | #note for csv data logging input data must be in json format
215 | if options["CSV"]:
216 |     csv_flag=True
217 |     print("Logging CSV format")
218 | if options["CSV"]==False and options["JSON"]==False:
219 |     print("logging plain data")
220 | if options["storechangesonly"]:
221 |     print("starting storing only changed data")
222 | else :
223 |     print("starting storing all data")
224 |     
225 | ##
226 | #Log_worker_flag=True
227 | 
228 | t = threading.Thread(target=log_worker) #start logger
229 | t.start() #start logging thread
230 | ###
231 | 
232 | client.last_message=dict()
233 | client.q=q #make queue available as part of client
234 | 
235 | 
236 | print("topics",options["topics"])
237 | 
238 | try:
239 |     res=client.connect(client.broker,client.port)      #connect to broker
240 |     client.loop_start() #start loop
241 | 
242 | except:
243 |     print("connection to ",client.broker," failed")
244 |     logging.debug("connection to ",client.broker," failed")
245 |     raise SystemExit("connection failed")
246 | print("connected")
247 | try:
248 |     while True:
249 |         time.sleep(1)
250 |         pass
251 | 
252 | except KeyboardInterrupt:
253 |     print("interrrupted by keyboard")
254 | print("end")
255 | client.loop_stop() #start loop
256 | Log_worker_flag=False #stop logging thread
257 | time.sleep(5)
258 | 
259 | 


--------------------------------------------------------------------------------
/mqttdatalogger.py:
--------------------------------------------------------------------------------
  1 | #!c:\python34\python.exe
  2 | #!/usr/bin/env python
  3 | #If Running in Windows use top line and edit according to your python
  4 | #location and version. If running on Linux delete the top line.
  5 | ###demo code provided by Steve Cope at www.steves-internet-guide.com
  6 | ##email steve@steves-internet-guide.com
  7 | ###Free to use for any purpose
  8 | """
  9 | This will log messages to file.Los time,message and topic as JSON data
 10 | """
 11 | #updated 18-oct-2023
 12 | mqttclient_log=False #MQTT client logs showing messages
 13 | Log_worker_flag=True
 14 | import paho.mqtt.client as mqtt
 15 | import json
 16 | import os
 17 | import time
 18 | import sys, getopt,random
 19 | import logging
 20 | import mlogger as mlogger
 21 | import threading
 22 | from queue import Queue
 23 | from command import command_input
 24 | import command
 25 | import sys
 26 | print("Python version is", sys.version_info)
 27 | 
 28 | q=Queue()
 29 | 
 30 | ##helper functions
 31 | def convert(t):
 32 |     d=""
 33 |     for c in t:  # replace all chars outside BMP with a !
 34 |             d =d+(c if ord(c) < 0x10000 else '!')
 35 |     return(d)
 36 | ###
 37 | 
 38 | class MQTTClient(mqtt.Client):#extend the paho client class
 39 |    run_flag=False #global flag used in multi loop
 40 |    def __init__(self,cname,**kwargs):
 41 |       super(MQTTClient, self).__init__(cname,**kwargs)
 42 |       self.last_pub_time=time.time()
 43 |       self.topic_ack=[] #used to track subscribed topics
 44 |       self.run_flag=True
 45 |       self.submitted_flag=False #used for connections
 46 |       self.subscribe_flag=False
 47 |       self.bad_connection_flag=False
 48 |       self.bad_count=0
 49 |       self.connected_flag=False
 50 |       self.connect_flag=False #used in multi loop
 51 |       self.disconnect_flag=False
 52 |       self.disconnect_time=0.0
 53 |       self.pub_msg_count=0
 54 |       self.pub_flag=False
 55 |       self.sub_topic=""
 56 |       #self.sub_topics=["tele/#","stat/#"] #multiple topics
 57 |       self.sub_qos=0
 58 |       self.devices=[]
 59 |       self.broker="mqtt2.home"
 60 |       self.port=1883
 61 |       self.keepalive=60
 62 |       self.run_forever=False
 63 |       self.cname=""
 64 |       self.delay=10 #retry interval
 65 |       self.retry_time=time.time()
 66 | 
 67 | def Initialise_clients(cname,mqttclient_log=False,cleansession=True,flags=""):
 68 |     #flags set
 69 |    print("initialising clients")
 70 |    logging.info("initialising clients")
 71 |    client= MQTTClient(cname,clean_session=cleansession)
 72 |    client.cname=cname
 73 |    client.on_connect= on_connect        #attach function to callback
 74 |    client.on_message=on_message        #attach function to callback
 75 |    #client.on_disconnect=on_disconnect
 76 |    #client.on_subscribe=on_subscribe
 77 |    if mqttclient_log:
 78 |       client.on_log=on_log
 79 |    return client
 80 | 
 81 | def on_connect(client, userdata, flags, rc):
 82 |    """
 83 |    set the bad connection flag for rc >0, Sets onnected_flag if connected ok
 84 |    also subscribes to topics
 85 |    """
 86 |    logging.debug("Connected flags"+str(flags)+"result code "\
 87 |     +str(rc)+"client1_id")
 88 |    if rc==0:
 89 |       
 90 |       client.connected_flag=True #old clients use this
 91 |       client.bad_connection_flag=False
 92 |       if client.sub_topic!="": #single topic
 93 |          print("subscribing "+str(client.sub_topic))
 94 |          print("subscribing in on_connect ")
 95 |          topic=client.sub_topics
 96 |          if client.sub_qos!=0:
 97 |             client.subscribe(topic,qos)
 98 |       elif client.sub_topics!="":
 99 |          print("subscribing in on_connect multiple",client.sub_topics)
100 |          client.subscribe(client.sub_topics)
101 | 
102 |    else:
103 |      print("set bad connection flag")
104 |      client.bad_connection_flag=True #
105 |      client.bad_count +=1
106 |      client.connected_flag=False #
107 | def on_message(client,userdata, msg):
108 |     topic=msg.topic
109 |     m_decode=str(msg.payload.decode("utf-8","ignore"))
110 |     message_handler(client,m_decode,topic)
111 |     #print("message received")
112 |     
113 | def message_handler(client,msg,topic):
114 |     data=dict()
115 |     tnow=time.ctime()
116 |     #m=time.asctime(tnow)+" "+topic+" "+msg
117 |     try:
118 |         msg=json.loads(msg)#convert to Javascript before saving
119 |         #print("json data")
120 |     except:
121 |         pass
122 |         #print("not already json")
123 | 
124 |     data["time"]=tnow
125 |     data["time_ms"]=time.time()
126 |     data["topic"]=topic
127 |     if csv_flag:
128 |         try:
129 |             keys=msg.keys()
130 |             for key in keys:
131 |                 data[key]=msg[key]
132 |         except:
133 |             data["message"]=msg 
134 |     else:             
135 |         data["message"]=msg
136 |     
137 |     #data["message"]=msg
138 |     if command.options["storechangesonly"]:
139 |         if has_changed(client,topic,msg):
140 |             client.q.put(data) #put messages on queue
141 |     else:
142 |         client.q.put(data) #put messages on queue
143 | 
144 | def has_changed(client,topic,msg):
145 |     topic2=topic.lower()
146 |     if topic2.find("control")!=-1:
147 |         return False
148 |     if topic in client.last_message:
149 |         if client.last_message[topic]==msg:
150 |             return False
151 |     client.last_message[topic]=msg
152 |     return True
153 | ###
154 | def log_worker():
155 |     """runs in own thread to log data from queue"""
156 |     while Log_worker_flag:
157 |         time.sleep(0.01)
158 |         while not q.empty():
159 |             results = q.get()
160 |             if results is None:
161 |                 continue
162 |  
163 |             if csv_flag:
164 |                 log.log_csv(results)
165 |             else:
166 |                 log.log_json(results)
167 |   
168 |             
169 |             #print("message saved ",results["message"])
170 |     log.close_file()
171 | 
172 | # MAIN PROGRAM
173 | options=command.options
174 | 
175 | if __name__ == "__main__" and len(sys.argv)>=2:
176 |     options=command_input(options)
177 | else:
178 |     print("Need broker name and topics to continue.. exiting")
179 |     #raise SystemExit(1)
180 | 
181 | 
182 | if not options["cname"]:
183 |     r=random.randrange(1,10000)
184 |     cname="logger-"+str(r)
185 | else:
186 |     cname="logger-"+str(options["cname"])
187 | log_dir=options["log_dir"]
188 | log_records=options["log_records"]
189 | number_logs=options["number_logs"]
190 | log=mlogger.m_logger(log_dir,log_records,number_logs) #create log object
191 | print("Log Directory =",log_dir)
192 | print("Log records per log =",log_records)
193 | if number_logs==0:
194 |     print("Max logs = Unlimited")
195 | else:
196 |     print("Max logs  =",number_logs)
197 |     
198 | 
199 | logging.info("creating client"+cname)
200 | 
201 | client=Initialise_clients(cname,mqttclient_log,False)#create and initialise client object
202 | if options["username"] !="":
203 |     client.username_pw_set(options["username"], options["password"])
204 | 
205 | client.sub_topics=options["topics"]
206 | client.broker=options["broker"]
207 | client.port=options["port"]
208 | json_flag=False
209 | csv_flag=False
210 | 
211 | if options["JSON"]:
212 |     print("Logging JSON format")
213 |     json_flag=True
214 | #note for csv data logging input data must be in json format
215 | if options["CSV"]:
216 |     csv_flag=True
217 |     print("Logging CSV format")
218 | if options["CSV"]==False and options["JSON"]==False:
219 |     print("logging plain data")
220 | if options["storechangesonly"]:
221 |     print("starting storing only changed data")
222 | else :
223 |     print("starting storing all data")
224 |     
225 | ##
226 | #Log_worker_flag=True
227 | 
228 | t = threading.Thread(target=log_worker) #start logger
229 | t.start() #start logging thread
230 | ###
231 | 
232 | client.last_message=dict()
233 | client.q=q #make queue available as part of client
234 | 
235 | 
236 | print("topics",options["topics"])
237 | 
238 | try:
239 |     res=client.connect(client.broker,client.port)      #connect to broker
240 |     client.loop_start() #start loop
241 | 
242 | except:
243 |     print("connection to ",client.broker," failed")
244 |     logging.debug("connection to ",client.broker," failed")
245 |     raise SystemExit("connection failed")
246 | print("connected")
247 | try:
248 |     while True:
249 |         time.sleep(1)
250 |         pass
251 | 
252 | except KeyboardInterrupt:
253 |     print("interrrupted by keyboard")
254 | print("end")
255 | client.loop_stop() #start loop
256 | Log_worker_flag=False #stop logging thread
257 | time.sleep(5)
258 | 
259 | 


--------------------------------------------------------------------------------