Logger

intersectingValidation.py

@@ -1,8 +1,13 @@
 import json
+import logging
+from logging.config import fileConfig
+
 import pandas as pd
 from shapely.geometry import LineString, Point, Polygon, MultiPoint
 import time
 
+from timerLog import timecall
+
 def readJsonFile(path):
     '''
     To read Jsonfile with the given path
@@ -22,7 +27,7 @@ def readJsonFile(path):
         dataDict = json.load(data_json)
         return dataDict
 
-
+@timecall(log_name='IntersectingValidation',log_level=logging.INFO,immediate=False,messages="step5")
 def geometryFormat(geometryDatarow):
     if (geometryDatarow[0]["type"] == "LineString"):
         try:
@@ -41,13 +46,14 @@ def geometryFormat(geometryDatarow):
             return "invalid"
 
 
+@timecall(log_name='IntersectingValidation',log_level=logging.INFO,immediate=False,messages="step4")
 def indexInvalidGeometryType(geometryData):
     dataInvalidGeoJsonFormat = geometryData.apply(geometryFormat, axis=1)
     ind_drop = dataInvalidGeoJsonFormat[dataInvalidGeoJsonFormat.apply(lambda row: row == 'invalid')].index
 
     return ind_drop
 
-
+@timecall(log_name='IntersectingValidation',log_level=logging.INFO,immediate=False,messages="step3")
 def brunnelcheck(x, skipTag):
     if (skipTag in x[0].keys()):
         if (x[0]["brunnel"] == None):
@@ -57,6 +63,7 @@ def brunnelcheck(x, skipTag):
     else:
         return False
 
+@timecall(log_name='IntersectingValidation',log_level=logging.INFO,immediate=False,messages="step2")
 def geojsonWrite(path, invalidWays, originalGeoJsonFile):
     '''
     To write the given invalid ways in the GeoJson format
@@ -80,8 +87,11 @@ def geojsonWrite(path, invalidWays, originalGeoJsonFile):
     with open(path.split('.')[0] + '.geojson', 'w') as fp:
         json.dump(invalidPaths, fp, indent=4)
 
-
+@timecall(log_name='IntersectingValidation',log_level=logging.INFO,immediate=False,messages="step1")
 def intersectLineStringInValidFormat(geoJSONdata, skipTag, cf):
+    fileConfig('logging_config.ini')
+    IntersectingValidationLogger = logging.getLogger("IntersectingValidation")
+
     featuresData = pd.DataFrame(geoJSONdata["features"])
     geometryData = pd.DataFrame(featuresData["geometry"])
     propertyData = pd.DataFrame(featuresData["properties"])
@@ -93,7 +103,6 @@ def intersectLineStringInValidFormat(geoJSONdata, skipTag, cf):
     start_time = time.time()
 
     invalidGeometryIndex = indexInvalidGeometryType(geometryData).values.tolist()
-    print("--- %s seconds ---" % (time.time() - start_time))
 
     for counter in range(len(geoJSONdata["features"])):
         if counter in invalidGeometryIndex:
@@ -104,7 +113,8 @@ def intersectLineStringInValidFormat(geoJSONdata, skipTag, cf):
     geometryDataFormatCopy = geometryDataFormat.copy()
 
     for rowIdI, wayI in geometryDataFormat.iteritems():
-        if (rowIdI == len(geometryDataFormat) / 2):
+        if (rowIdI == int(len(geometryDataFormat) / 2)):
+            IntersectingValidationLogger.info("function:intersectLineStringInValidFormat, step1, 0 calls, 0 seconds, 0.000 seconds per call,half way through... please wait")
             print("half way through... please wait")
 
         if (wayI == "invalid" or rowIdI in brunnelExist):
@@ -139,7 +149,7 @@ def intersectLineStringInValidFormat(geoJSONdata, skipTag, cf):
                             {"type": "Feature", "geometry": {"type": "Point", "coordinates": appendPoints[0]}})
 
                 else:
-                    print("Invalid format Support not given yet")
+                    IntersectingValidationLogger.error("Invalid format Support not given yet")
                     exit(0)
 
                 if (len(violatingWayFeatures) == 0):

log/logfile.log

@@ -0,0 +1,22 @@
+2020-08-28 23:36:14,693,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Number of geojson files read : 2
+2020-08-28 23:36:14,693,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call,Processing File : ms_campus_nodes.geojson  ms_campus_ts.geojson
+2020-08-28 23:36:22,698,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Running intersectingvalidation to see if any ways which are intersecting have a missing intersecting node
+2020-08-28 23:36:45,509,  IntersectingValidation, INFO    , function:intersectLineStringInValidFormat, step1, 0 calls, 0 seconds, 0.000 seconds per call,half way through... please wait
+2020-08-28 23:36:54,073,  main, INFO    , function:main, step 1,  1 calls, 0 seconds, 0.000 seconds per call, Performing Exploratory Data Analysis
+2020-08-28 23:36:54,262,  EDA, INFO    , function: subgraph_eda, step 3, 1 calls, 0.00872 seconds, 0.00872 seconds per call, Number of ways in the file : 4189 Number of isolated ways: 17 Number of Connected ways: 4172 Number of Connected Components : 52
+2020-08-28 23:36:54,515,  main, INFO    , function:main, step 1, 1 calls, 0 seconds, 0.000 seconds per call, Performing all validations
+2020-08-28 23:36:54,518,  utildata, INFO    , function: get_one_node_ways, step1, 1 calls, 0.000 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:54,518,  utildata, INFO    , function: get_coord_df, step2, 1 calls, 7.810 seconds, 7.810 seconds per call, No Comments
+2020-08-28 23:36:54,518,  utildata, INFO    , function: split_ways_geojson_file, step3, 1 calls, 0.140 seconds, 0.140 seconds per call, No Comments
+2020-08-28 23:36:54,518,  utildata, INFO    , function: get_isolated_ways, step4, 1 calls, 0.006 seconds, 0.006 seconds per call, No Comments
+2020-08-28 23:36:54,518,  utildata, INFO    , function: get_coord_dict, step5, 1 calls, 0.031 seconds, 0.031 seconds per call, No Comments
+2020-08-28 23:36:54,518,  utildata, INFO    , function: get_coords_list, step6, 2 calls, 0.001 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:54,518,  EDA, INFO    , function: get_invalidNodes, step1, 1 calls, 0.003 seconds, 0.003 seconds per call, No Comments
+2020-08-28 23:36:54,518,  EDA, INFO    , function: get_way_from_subgraph, step2, 1 calls, 0.006 seconds, 0.006 seconds per call, No Comments
+2020-08-28 23:36:54,518,  EDA, INFO    , function: subgraph_eda, step3, 1 calls, 0.266 seconds, 0.266 seconds per call, No Comments
+2020-08-28 23:36:54,519,  EDA, INFO    , function: plot_nodes_vs_ways, step4, 1 calls, 0.176 seconds, 0.176 seconds per call, No Comments
+2020-08-28 23:36:54,519,  IntersectingValidation, INFO    , function: intersectLineStringInValidFormat, step1, 1 calls, 31.374 seconds, 31.374 seconds per call, No Comments
+2020-08-28 23:36:54,519,  IntersectingValidation, INFO    , function: geojsonWrite, step2, 3 calls, 0.003 seconds, 0.001 seconds per call, No Comments
+2020-08-28 23:36:54,519,  IntersectingValidation, INFO    , function: brunnelcheck, step3, 4189 calls, 0.010 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:54,519,  IntersectingValidation, INFO    , function: indexInvalidGeometryType, step4, 1 calls, 0.060 seconds, 0.060 seconds per call, No Comments
+2020-08-28 23:36:54,519,  IntersectingValidation, INFO    , function: geometryFormat, step5, 8378 calls, 0.071 seconds, 0.000 seconds per call, No Comments

log/logfile.old.20200828-233531.log

@@ -0,0 +1,22 @@
+2020-08-28 23:34:26,176,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Number of geojson files read : 2
+2020-08-28 23:34:26,176,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call,Processing File : ms_campus_nodes.geojson  ms_campus_ts.geojson
+2020-08-28 23:34:34,075,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Running intersectingvalidation to see if any ways which are intersecting have a missing intersecting node
+2020-08-28 23:34:58,265,  IntersectingValidation, INFO    , function:intersectLineStringInValidFormat, step1, 0 calls, 0 seconds, 0.000 seconds per call,half way through... please wait
+2020-08-28 23:35:08,069,  main, INFO    , function:main, step 1,  1 calls, 0 seconds, 0.000 seconds per call, Performing Exploratory Data Analysis
+2020-08-28 23:35:08,256,  EDA, INFO    , function: subgraph_eda, step 3, 1 calls, 0.00878 seconds, 0.00878 seconds per call, Number of ways in the file : 4189 Number of isolated ways: 17 Number of Connected ways: 4172 Number of Connected Components : 52
+2020-08-28 23:35:08,518,  main, INFO    , function:main, step 1, 1 calls, 0 seconds, 0.000 seconds per call, Performing all validations
+2020-08-28 23:35:08,522,  utildata, INFO    , function: get_one_node_ways, step1, 1 calls, 0.000 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:35:08,522,  utildata, INFO    , function: get_coord_df, step2, 1 calls, 7.702 seconds, 7.702 seconds per call, No Comments
+2020-08-28 23:35:08,522,  utildata, INFO    , function: split_ways_geojson_file, step3, 1 calls, 0.142 seconds, 0.142 seconds per call, No Comments
+2020-08-28 23:35:08,522,  utildata, INFO    , function: get_isolated_ways, step4, 1 calls, 0.006 seconds, 0.006 seconds per call, No Comments
+2020-08-28 23:35:08,522,  utildata, INFO    , function: get_coord_dict, step5, 1 calls, 0.031 seconds, 0.031 seconds per call, No Comments
+2020-08-28 23:35:08,522,  utildata, INFO    , function: get_coords_list, step6, 2 calls, 0.001 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:35:08,522,  EDA, INFO    , function: get_invalidNodes, step1, 1 calls, 0.003 seconds, 0.003 seconds per call, No Comments
+2020-08-28 23:35:08,522,  EDA, INFO    , function: get_way_from_subgraph, step2, 1 calls, 0.007 seconds, 0.007 seconds per call, No Comments
+2020-08-28 23:35:08,522,  EDA, INFO    , function: subgraph_eda, step3, 1 calls, 0.275 seconds, 0.275 seconds per call, No Comments
+2020-08-28 23:35:08,522,  EDA, INFO    , function: plot_nodes_vs_ways, step4, 1 calls, 0.174 seconds, 0.174 seconds per call, No Comments
+2020-08-28 23:35:08,522,  IntersectingValidation, INFO    , function: intersectLineStringInValidFormat, step1, 1 calls, 33.994 seconds, 33.994 seconds per call, No Comments
+2020-08-28 23:35:08,522,  IntersectingValidation, INFO    , function: geojsonWrite, step2, 3 calls, 0.003 seconds, 0.001 seconds per call, No Comments
+2020-08-28 23:35:08,522,  IntersectingValidation, INFO    , function: brunnelcheck, step3, 4189 calls, 0.010 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:35:08,522,  IntersectingValidation, INFO    , function: indexInvalidGeometryType, step4, 1 calls, 0.058 seconds, 0.058 seconds per call, No Comments
+2020-08-28 23:35:08,522,  IntersectingValidation, INFO    , function: geometryFormat, step5, 8378 calls, 0.069 seconds, 0.000 seconds per call, No Comments

log/logfile.old.20200828-233614.log

@@ -0,0 +1,22 @@
+2020-08-28 23:35:31,145,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Number of geojson files read : 2
+2020-08-28 23:35:31,145,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call,Processing File : ms_campus_nodes.geojson  ms_campus_ts.geojson
+2020-08-28 23:35:38,864,  main, INFO    , function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Running intersectingvalidation to see if any ways which are intersecting have a missing intersecting node
+2020-08-28 23:36:01,357,  IntersectingValidation, INFO    , function:intersectLineStringInValidFormat, step1, 0 calls, 0 seconds, 0.000 seconds per call,half way through... please wait
+2020-08-28 23:36:10,648,  main, INFO    , function:main, step 1,  1 calls, 0 seconds, 0.000 seconds per call, Performing Exploratory Data Analysis
+2020-08-28 23:36:10,836,  EDA, INFO    , function: subgraph_eda, step 3, 1 calls, 0.00868 seconds, 0.00868 seconds per call, Number of ways in the file : 4189 Number of isolated ways: 17 Number of Connected ways: 4172 Number of Connected Components : 52
+2020-08-28 23:36:11,093,  main, INFO    , function:main, step 1, 1 calls, 0 seconds, 0.000 seconds per call, Performing all validations
+2020-08-28 23:36:11,096,  utildata, INFO    , function: get_one_node_ways, step1, 1 calls, 0.000 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:11,096,  utildata, INFO    , function: get_coord_df, step2, 1 calls, 7.520 seconds, 7.520 seconds per call, No Comments
+2020-08-28 23:36:11,096,  utildata, INFO    , function: split_ways_geojson_file, step3, 1 calls, 0.143 seconds, 0.143 seconds per call, No Comments
+2020-08-28 23:36:11,096,  utildata, INFO    , function: get_isolated_ways, step4, 1 calls, 0.006 seconds, 0.006 seconds per call, No Comments
+2020-08-28 23:36:11,096,  utildata, INFO    , function: get_coord_dict, step5, 1 calls, 0.031 seconds, 0.031 seconds per call, No Comments
+2020-08-28 23:36:11,097,  utildata, INFO    , function: get_coords_list, step6, 2 calls, 0.001 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:11,097,  EDA, INFO    , function: get_invalidNodes, step1, 1 calls, 0.003 seconds, 0.003 seconds per call, No Comments
+2020-08-28 23:36:11,097,  EDA, INFO    , function: get_way_from_subgraph, step2, 1 calls, 0.008 seconds, 0.008 seconds per call, No Comments
+2020-08-28 23:36:11,097,  EDA, INFO    , function: subgraph_eda, step3, 1 calls, 0.269 seconds, 0.269 seconds per call, No Comments
+2020-08-28 23:36:11,097,  EDA, INFO    , function: plot_nodes_vs_ways, step4, 1 calls, 0.175 seconds, 0.175 seconds per call, No Comments
+2020-08-28 23:36:11,097,  IntersectingValidation, INFO    , function: intersectLineStringInValidFormat, step1, 1 calls, 31.784 seconds, 31.784 seconds per call, No Comments
+2020-08-28 23:36:11,097,  IntersectingValidation, INFO    , function: geojsonWrite, step2, 3 calls, 0.003 seconds, 0.001 seconds per call, No Comments
+2020-08-28 23:36:11,097,  IntersectingValidation, INFO    , function: brunnelcheck, step3, 4189 calls, 0.010 seconds, 0.000 seconds per call, No Comments
+2020-08-28 23:36:11,097,  IntersectingValidation, INFO    , function: indexInvalidGeometryType, step4, 1 calls, 0.060 seconds, 0.060 seconds per call, No Comments
+2020-08-28 23:36:11,097,  IntersectingValidation, INFO    , function: geometryFormat, step5, 8378 calls, 0.073 seconds, 0.000 seconds per call, No Comments

logging_config.ini

@@ -0,0 +1,85 @@
+[loggers]
+keys=root,sLogger,EDA,main,format,read,write,plot,IntersectingValidation,utildata
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=fileFormatter,consoleFormatter
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_sLogger]
+level=DEBUG
+handlers=consoleHandler,fileHandler
+qualname=sLogger
+propagate=0
+
+[logger_EDA]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=EDA
+propagate=0
+
+[logger_utildata]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=utildata
+propagate=0
+
+[logger_IntersectingValidation]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=IntersectingValidation
+propagate=0
+
+[logger_main]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=main
+propagate=0
+
+[logger_format]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=format
+propagate=0
+
+[logger_read]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=read
+propagate=0
+
+[logger_write]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=write
+propagate=0
+
+[logger_plot]
+level=INFO
+handlers=consoleHandler,fileHandler
+qualname=write
+propagate=0
+
+[handler_consoleHandler]
+class=StreamHandler
+level=WARNING
+formatter=consoleFormatter
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=handlers.TimedRotatingFileHandler
+filemode='w+'
+level=DEBUG
+formatter=fileFormatter
+args=('log/logfile.log',)
+
+[formatter_fileFormatter]
+format=%(asctime)s,  %(name)s, %(levelname)-8s, %(message)s
+
+[formatter_consoleFormatter]
+format=%(levelname)s  %(message)s

main.py

@@ -1,13 +1,24 @@
 import argparse as ag
 import os
+from datetime import datetime
+
 from intersectingValidation import intersectLineStringInValidFormat
 from glob import glob
 from node_connectivity import plot_nodes_vs_ways, subgraph_eda, get_invalidNodes
 from config import DefaultConfigs
 from util_data import UtilData
 import ntpath
+import logging
+from logging.config import fileConfig
+import shutil
 
 if __name__ == '__main__':
+    filename1 = datetime.now().strftime("%Y%m%d-%H%M%S")
+    shutil.copy2('log/logfile.log', 'log/logfile.old.' + filename1 + '.log')
+    open('log/logfile.log', "w+").truncate(0)
+
+    fileConfig('logging_config.ini')
+    mainLogger = logging.getLogger("main")
     parser = ag.ArgumentParser()
     parser.add_argument("--inputPath", help="Relative input path to GeoJSON files",
                         default=os.path.join(os.getcwd(), "OSW\TestData\input"))
@@ -21,30 +32,43 @@
 
     json_files = glob(os.path.join(inputPath, "*.geojson"))
     if cf.file_filter:
-        print("Filtering")
         json_files = sorted([i for i in json_files if cf.file_filter in i])
-    print("Number of geojson files :", len(json_files))
+    print("Number of geojson files read :", len(json_files))
+    mainLogger.info(
+        "function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Number of geojson files read : %s",
+        len(json_files))
     nodes_files = sorted([x for x in json_files if 'node' in x])
     ways_files = sorted([x for x in json_files if 'node' not in x])
 
     for ind, (nodes_file, ways_file) in enumerate(zip(nodes_files, ways_files)):
-        print('Processing File : \n{}\n{}'.format(ntpath.basename(nodes_file), ntpath.basename(ways_file)))
+        print('Processing File : {} , {}'.format(ntpath.basename(nodes_file), ntpath.basename(ways_file)))
+        mainLogger.info(
+            'function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call,Processing File : {}  {}'.format(
+                ntpath.basename(nodes_file), ntpath.basename(ways_file)))
         utild = UtilData(nodes_file, ways_file, cf)
         if cf.validation == 'intersectingvalidation':
             print("--" * 10)
-            print("intersectingvalidation")
+            print(
+                "Running intersectingvalidation to check if any ways which are intersecting have a missing intersection node")
+            mainLogger.info(
+                "function:main, step 0, 0 calls, 0 seconds, 0.000 seconds per call, Running intersectingvalidation to see if any ways which are intersecting have a missing intersecting node")
+
             print("--" * 10)
             intersectLineStringInValidFormat(utild.ways_json, "brunnel", cf)
 
         if cf.do_eda:
             print("--" * 10)
-            print("eda")
+            print("Performing Exploratory Data Analysis")
+            mainLogger.info(
+                "function:main, step 1,  1 calls, 0 seconds, 0.000 seconds per call, Performing Exploratory Data Analysis")
             print("--" * 10)
             plot_nodes_vs_ways(utild, cf)
             subgraph_eda(utild, cf)
 
         if cf.do_all_validations:
             print("--" * 10)
-            print("all validations")
+            print("Performing Exploratory Data Analysis")
+            mainLogger.info(
+                "function:main, step 1, 1 calls, 0 seconds, 0.000 seconds per call, Performing all validations")
             print("--" * 10)
             get_invalidNodes(utild, cf)