mqtt-data-logger/mlogger.py at master · stevecope/mqtt-data-logger · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
###demo code provided by Steve Cope at www.steves-internet-guide.com
##email steve@steves-internet-guide.com
###Free to use for any purpose
"""
implements data logging class
"""
import time,os,json,logging

###############
class m_logger(object):
    """Class for logging data to a file. You can set the maximim number
of messages in a file the default is 5000. Setting number_logs to 0
means that there is no limit on the number of logs.When the file is full
a new file is created.Log files are stored under a root directory
and a sub directory that uses the timestamp for the directory name
Log file data is flushed immediately to disk so that data is not lost.
Data can be stored as plain text or in JSON format """
    def __init__(self,log_dir="mlogs",log_recs=5000,number_logs=0,csv_flag=False):
        self.log_dir=log_dir
        self.log_recs=log_recs
        self.number_logs=number_logs
        self.count=0
        self.log_dir=self.create_log_dir(self.log_dir)
        self.fo=self.get_log_name(self.log_dir,self.count)
        self.new_file_flag=0
        self.writecount=0
        self.timenow=time.time()
        self.flush_flag=True
        self.flush_time=2 #flush logs to disk every 2 seconds
        self.csv_flag = csv_flag
        self.columns=""

    def extract_columns(self,data):
        columns=""

        #data=flatten_dict(msg)
        for key in data:
            #print("key =",key)
            if columns =="":
                columns=key
            else:
                columns=columns+","+key
        #print(columns)
        return(columns)
    def extract_data(self,data):
        line_out=""
        for key in data:
            #print("here ",data[key])
            if line_out =="":
                line_out=str(data[key])
            else:
                line_out=line_out+","+str(data[key])
        #print(line_out)
        return(line_out)
    def __flushlogs(self): # write to disk
        self.fo.flush()
        #logging.info("flushing logs")
        os.fsync(self.fo.fileno())
        self.timenow=time.time()
    def __del__(self):
        if not self.fo.closed:
            print("closing log file")
            self.fo.close()
    def close_file(self):
        if not self.fo.closed:
            print("closing log file")
            self.fo.close()
    def create_log_dir(self,log_dir):
        """
        Function for creating new log directories
        using the timestamp for the name
        """
        self.t=time.localtime(time.time())
        self.time_stamp=(str(self.t[1])+"-"+str(self.t[2])+"-"+\
        str(self.t[3])+"-"+str(self.t[4]))
        logging.info("creating sub directory"+str(self.time_stamp))
        try:
            os.stat(self.log_dir)
        except:
            os.mkdir(self.log_dir)
        self.log_sub_dir=self.log_dir+"/"+self.time_stamp
        try:
            os.stat(self.log_sub_dir)
        except:
            os.mkdir(self.log_sub_dir)
        return(self.log_sub_dir)

    def get_log_name(self,log_dir,count):
        """get log files and directories"""
        self.log_numbr="{0:003d}".format(count)
        logging.info("s is"+str(self.log_numbr))
        self.file_name=self.log_dir+"/"+"log"+self.log_numbr +".log"
        logging.info("creating log file "+self.file_name)
        f = open(self.file_name,'w') #clears file if it exists
        f.close()
        f=open(self.file_name, 'a')
        return(f)
    def log_json(self,data): #data is a JavaScript object
        jdata=json.dumps(data)+"\n"
        self.log_data(jdata)
    def log_csv(self,data): #data is a JavaScript object that needs converting
        #print("logging csv ",self.writecount)
        if self.writecount==0:
            self.columns=self.extract_columns(data)+"\n"
            csv_flag=True
            self.log_data(self.columns)
        else:
            csv_data=self.extract_data(data)+"\n"
            self.log_data(csv_data)
    def log_data(self, data):
        #print("logging data")
        self.data=data
        try:
            self.fo.write(data)
            self.writecount+=1
            self.__flushlogs()
            if self.writecount>=self.log_recs:
                self.count+=1 #counts number of logs
                if self.count>self.number_logs and self.number_logs !=0 :
                      logging.info("too many logs: starting from 0")
                      self.count=0 #reset
                self.fo=self.get_log_name(self.log_dir,self.count)
                self.writecount=0
        except BaseException as e:
            logging.error("Error on_data: %s" % str(e))
            return False
        return True