Validator quality update (#25)

* Change testing trace
* Removed version definition PyYaml
* Use trace with 50 messages
* Update osi version
* Added black code formatter
* Added vulture and removed unused variables
* Removed redundant code and added explicit folders for vulture

Author: vkresch

.travis.yml

@@ -34,6 +34,14 @@ script:
     - python -m site
     - protoc --version
 
+    # Install black code formatter and check if it is applied
+    - pip install black
+    - black --check --exclude "(open-simulation-interface|proto2cpp)" .
+
+    # Install vulture and search for dead code with 100% confidence
+    - pip install vulture
+    - vulture *.py tests/ osivalidator/ --min-confidence 100
+
     # Install osi-validation
     - cd open-simulation-interface
     - pip install .
@@ -43,9 +51,6 @@ script:
     # Generate parsed rules
     - python rules2yml.py -d rules
 
-    # Check if rule syntax in osi is correct
-    # - python test_cases.py
-
     # Show validator usage
     - osivalidator -h
 

data/small_test.txt.lzma

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b572e8d38c1ff03c6e58168e99be5560ca310c1052a6e9c20f74432b46547314
-size 130010
+oid sha256:97d4efda5888898a82044e2093ee605bade75290038afaf9a03d7bceecc91d98
+size 442

open-simulation-interface

@@ -93,8 +93,10 @@ def get_field(self, field_name):
 
         field = getattr(self.value, field_name)
 
-        if (hasattr(self.value, 'DESCRIPTOR') and
-                self.value.DESCRIPTOR.fields_by_name[field_name].label == 3):
+        if (
+            hasattr(self.value, "DESCRIPTOR")
+            and self.value.DESCRIPTOR.fields_by_name[field_name].label == 3
+        ):
             return [
                 LinkedProtoField(u_field, parent=self, name=field_name)
                 for u_field in field

osivalidator/linked_proto_field.py

@@ -21,19 +21,18 @@ class OSIDoxygenXML:
     def __init__(self):
         dir_path = os.path.dirname(os.path.realpath(__file__))
         osivalidator_path = os.path.dirname(dir_path)
-        self.osi_path = os.path.join(
-            osivalidator_path, 'open-simulation-interface')
-        self.osi_doc_path = os.path.join(self.osi_path, 'doc')
+        self.osi_path = os.path.join(osivalidator_path, "open-simulation-interface")
+        self.osi_doc_path = os.path.join(self.osi_path, "doc")
 
-        proto2cpp_path = os.path.join(osivalidator_path, 'proto2cpp')
+        proto2cpp_path = os.path.join(osivalidator_path, "proto2cpp")
         self.proto2cpp_file_path = os.path.join(proto2cpp_path, "proto2cpp.py")
 
     def generate_osi_doxygen_xml(self):
         """
         Generate the Doxygen XML documentation in the OSI path
         """
 
-        configuration = f'''
+        configuration = f"""
         PROJECT_NAME           = osi-validation
         INPUT                  = {self.osi_path}
         OUTPUT_DIRECTORY       = {self.osi_doc_path}
@@ -48,10 +47,10 @@ def generate_osi_doxygen_xml(self):
         XML_PROGRAMLISTING     = NO
         ALIASES                = rules="<pre class=\"rules\">"
         ALIASES               += endrules="</pre>"
-        '''
+        """
 
-        doxyfile_path = os.path.join(self.osi_path, 'Doxyfile_validation')
-        doxyfile = open(doxyfile_path, 'w')
+        doxyfile_path = os.path.join(self.osi_path, "Doxyfile_validation")
+        doxyfile = open(doxyfile_path, "w")
         doxyfile.write(configuration)
         doxyfile.close()
 
@@ -65,7 +64,7 @@ def get_files(self):
         """
         Return the path of the fields in OSI
         """
-        return glob.glob(os.path.join(self.osi_path, 'doc', 'xml', '*.xml'))
+        return glob.glob(os.path.join(self.osi_path, "doc", "xml", "*.xml"))
 
     def parse_rules(self):
         """
@@ -78,19 +77,20 @@ def parse_rules(self):
         for xml_file in xml_files:
             tree = ET.parse(xml_file)
             memberdefs = tree.findall(
-                "./compounddef/sectiondef/memberdef[@kind='variable']")
+                "./compounddef/sectiondef/memberdef[@kind='variable']"
+            )
             for memberdef in memberdefs:
-                attr_path = memberdef.findtext(
-                    'definition').split()[-1].split('::')
-                if attr_path[0] != 'osi3':
+                attr_path = memberdef.findtext("definition").split()[-1].split("::")
+                if attr_path[0] != "osi3":
                     continue
                 attr_path.pop(0)
                 attr_rule = memberdef.findtext(
-                    './detaileddescription//preformatted[last()]')
+                    "./detaileddescription//preformatted[last()]"
+                )
                 if not attr_rule:
                     continue
 
-                rules_lines = attr_rule.split('\n')
+                rules_lines = attr_rule.split("\n")
 
                 for line_no, line in enumerate(rules_lines):
                     if line.find(":") == -1 and line:
@@ -100,14 +100,13 @@ def parse_rules(self):
 
                 try:
                     dict_rules = yaml.safe_load(attr_rule)
-                except (yaml.parser.ParserError,
-                        yaml.parser.ScannerError) as error:
+                except (yaml.parser.ParserError, yaml.parser.ScannerError) as error:
                     print(attr_path, attr_rule, error)
                 else:
                     rules.append((attr_path, dict_rules))
         return rules
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     osidx = OSIDoxygenXML()
     osidx.generate_osi_doxygen_xml()

osivalidator/osi_doxygen_xml.py

@@ -13,70 +13,95 @@
 from osivalidator.osi_trace import OSITrace
 from osivalidator.osi_rules_checker import OSIRulesChecker
 
+
 def check_positive_int(value):
     ivalue = int(value)
     if ivalue < 0:
         raise argparse.ArgumentTypeError("%s is an invalid positive int value" % value)
     return ivalue
 
+
 def command_line_arguments():
     """ Define and handle command line interface """
 
     dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
     parser = argparse.ArgumentParser(
-        description='Validate data defined at the input',
-        prog='osivalidator')
-    parser.add_argument('data',
-                        help='Path to the file with OSI-serialized data.',
-                        type=str)
-    parser.add_argument('--rules', '-r',
-                        help='Directory with text files containig rules. ',
-                        default=os.path.join(dir_path, 'requirements-osi-3'),
-                        type=str)
-    parser.add_argument('--type', '-t',
-                        help='Name of the type used to serialize data.',
-                        choices=['SensorView', 'GroundTruth', 'SensorData'],
-                        default='SensorView',
-                        type=str,
-                        required=False)
-    parser.add_argument('--output', '-o',
-                        help='Output folder of the log files.',
-                        default='output_logs',
-                        type=str,
-                        required=False)
-    parser.add_argument('--timesteps',
-                        help='Number of timesteps to analyze. If -1, all.',
-                        type=int,
-                        default=-1,
-                        required=False)
-    parser.add_argument('--debug',
-                        help='Set the debug mode to ON.',
-                        action="store_true")
-    parser.add_argument('--verbose', '-v',
-                        help='Set the verbose mode to ON.',
-                        action="store_true")
-    parser.add_argument('--parallel', '-p',
-                        help='Set parallel mode to ON.',
-                        default=False,
-                        required=False,
-                        action="store_true")
-    parser.add_argument('--format', '-f',
-                        help='Set the format type of the trace.',
-                        choices=['separated', None],
-                        default=None,
-                        type=str,
-                        required=False)
-    parser.add_argument('--blast', '-bl',
-                        help='Set the in-memory storage count of OSI messages during validation.',
-                        default=500,
-                        type=check_positive_int,
-                        required=False)
-    parser.add_argument('--buffer', '-bu',
-                        help='Set the buffer size to retrieve OSI messages from trace file. Set it to 0 if you do not want to use buffering at all.',
-                        default=1000000,
-                        type=check_positive_int,
-                        required=False)
+        description="Validate data defined at the input", prog="osivalidator"
+    )
+    parser.add_argument(
+        "data", help="Path to the file with OSI-serialized data.", type=str
+    )
+    parser.add_argument(
+        "--rules",
+        "-r",
+        help="Directory with text files containig rules. ",
+        default=os.path.join(dir_path, "requirements-osi-3"),
+        type=str,
+    )
+    parser.add_argument(
+        "--type",
+        "-t",
+        help="Name of the type used to serialize data.",
+        choices=["SensorView", "GroundTruth", "SensorData"],
+        default="SensorView",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="Output folder of the log files.",
+        default="output_logs",
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--timesteps",
+        help="Number of timesteps to analyze. If -1, all.",
+        type=int,
+        default=-1,
+        required=False,
+    )
+    parser.add_argument(
+        "--debug", help="Set the debug mode to ON.", action="store_true"
+    )
+    parser.add_argument(
+        "--verbose", "-v", help="Set the verbose mode to ON.", action="store_true"
+    )
+    parser.add_argument(
+        "--parallel",
+        "-p",
+        help="Set parallel mode to ON.",
+        default=False,
+        required=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--format",
+        "-f",
+        help="Set the format type of the trace.",
+        choices=["separated", None],
+        default=None,
+        type=str,
+        required=False,
+    )
+    parser.add_argument(
+        "--blast",
+        "-bl",
+        help="Set the in-memory storage count of OSI messages during validation.",
+        default=500,
+        type=check_positive_int,
+        required=False,
+    )
+    parser.add_argument(
+        "--buffer",
+        "-bu",
+        help="Set the buffer size to retrieve OSI messages from trace file. Set it to 0 if you do not want to use buffering at all.",
+        default=1000000,
+        type=check_positive_int,
+        required=False,
+    )
 
     return parser.parse_args()
 
@@ -88,10 +113,11 @@ def command_line_arguments():
 LOGGER = OSIValidatorLogger()
 VALIDATION_RULES = OSIRules()
 ID_TO_TS = MANAGER.dict()
-BAR_SUFFIX = '%(index)d/%(max)d [%(elapsed_td)s]'
-BAR = Bar('', suffix=BAR_SUFFIX)
+BAR_SUFFIX = "%(index)d/%(max)d [%(elapsed_td)s]"
+BAR = Bar("", suffix=BAR_SUFFIX)
 MESSAGE_CACHE = MANAGER.dict()
 
+
 def main():
     """Main method"""
 
@@ -112,7 +138,12 @@ def main():
     # Read data
     print("Reading data ...")
     DATA = OSITrace(buffer_size=args.buffer)
-    DATA.from_file(path=args.data, type_name=args.type, max_index=args.timesteps, format_type=args.format)
+    DATA.from_file(
+        path=args.data,
+        type_name=args.type,
+        max_index=args.timesteps,
+        format_type=args.format,
+    )
 
     # Collect Validation Rules
     print("Collect validation rules ...")
@@ -138,7 +169,7 @@ def main():
 
         # Increment the max-timestep to analyze
         max_timestep_blast += args.blast
-        first_of_blast = (max_timestep_blast-args.blast)
+        first_of_blast = max_timestep_blast - args.blast
         last_of_blast = min(max_timestep_blast, max_timestep)
 
         # Cache messages
@@ -148,9 +179,9 @@ def main():
         if args.parallel:
             # Launch parallel computation
             # Recreate the pool
-            try:                
+            try:
                 pool = Pool()
-                pool.map(process_timestep, range(first_of_blast, last_of_blast)) 
+                pool.map(process_timestep, range(first_of_blast, last_of_blast))
 
             except Exception as e:
                 print(str(e))
@@ -163,7 +194,7 @@ def main():
             try:
                 for i in range(first_of_blast, last_of_blast):
                     process_timestep(i)
-                    
+
             except Exception as e:
                 print(str(e))
 
@@ -176,6 +207,7 @@ def main():
     # Synthetize
     LOGGER.synthetize_results_from_sqlite()
 
+
 def close_pool(pool):
     """Cleanly close a pool to free the memory"""
     pool.close()
@@ -192,7 +224,7 @@ def process_timestep(timestep):
 
     LOGGER.log_messages[timestep] = []
     LOGGER.debug_messages[timestep] = []
-    LOGGER.info(None, f'Analyze message of timestamp {timestamp}', False)
+    LOGGER.info(None, f"Analyze message of timestamp {timestamp}", False)
 
     # Check if timestamp already exists
     if timestamp in TIMESTAMP_ANALYZED:
@@ -202,8 +234,9 @@ def process_timestep(timestep):
     BAR.goto(len(TIMESTAMP_ANALYZED))
 
     # Check common rules
-    getattr(rule_checker, 'is_valid')(
-        message, VALIDATION_RULES.get_rules().get_type(MESSAGE_TYPE.value))
+    getattr(rule_checker, "is_valid")(
+        message, VALIDATION_RULES.get_rules().get_type(MESSAGE_TYPE.value)
+    )
 
     LOGS.extend(LOGGER.log_messages[timestep])