Add a depth-max for test logs

Merge pull request #1150 from openfisca/depth-max

Author: bonjourmauko

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+### 35.9.0 [#1150](https://github.com/openfisca/openfisca-core/pull/1150)
+
+#### New Features
+
+- Introduce a maximal depth for compuation logs
+  - Allows for limiting the depth of the computation log chain
+
 ### 35.8.6 [#1145](https://github.com/openfisca/openfisca-core/pull/1145)
 
 #### Technical changes

openfisca_core/scripts/openfisca_command.py

@@ -38,7 +38,9 @@ def build_test_parser(parser):
         parser.add_argument('-p', '--pdb', action = 'store_true', default = False, help = "drop into debugger on failures or errors")
         parser.add_argument('--performance-graph', '--performance', action = 'store_true', default = False, help = "output a performance graph in a 'performance_graph.html' file")
         parser.add_argument('--performance-tables', action = 'store_true', default = False, help = "output performance CSV tables")
-        parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity")
+        parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity. If specified, output the entire calculation trace.")
+        parser.add_argument('-a', '--aggregate', action = 'store_true', default = False, help = "increase output verbosity to aggregate. If specified, output the avg, max, and min values of the calculation trace. This flag has no effect without --verbose.")
+        parser.add_argument('-d', '--max-depth', type = int, default = None, help = "set maximal verbosity depth. If specified, output the calculation trace up to the provided depth. This flag has no effect without --verbose.")
         parser.add_argument('-o', '--only-variables', nargs = '*', default = None, help = "variables to test. If specified, only test the given variables.")
         parser.add_argument('-i', '--ignore-variables', nargs = '*', default = None, help = "variables to ignore. If specified, do not test the given variables.")
 

openfisca_core/scripts/run_test.py

@@ -19,6 +19,8 @@ def main(parser):
         'performance_graph': args.performance_graph,
         'performance_tables': args.performance_tables,
         'verbose': args.verbose,
+        'aggregate': args.aggregate,
+        'max_depth': args.max_depth,
         'name_filter': args.name_filter,
         'only_variables': args.only_variables,
         'ignore_variables': args.ignore_variables,

openfisca_core/tools/test_runner.py

@@ -144,6 +144,8 @@ def runtest(self):
         period = self.test.get('period')
         max_spiral_loops = self.test.get('max_spiral_loops')
         verbose = self.options.get('verbose')
+        aggregate = self.options.get('aggregate')
+        max_depth = self.options.get('max_depth')
         performance_graph = self.options.get('performance_graph')
         performance_tables = self.options.get('performance_tables')
 
@@ -165,15 +167,15 @@ def runtest(self):
         finally:
             tracer = self.simulation.tracer
             if verbose:
-                self.print_computation_log(tracer)
+                self.print_computation_log(tracer, aggregate, max_depth)
             if performance_graph:
                 self.generate_performance_graph(tracer)
             if performance_tables:
                 self.generate_performance_tables(tracer)
 
-    def print_computation_log(self, tracer):
+    def print_computation_log(self, tracer, aggregate, max_depth):
         print("Computation log:")  # noqa T001
-        tracer.print_computation_log()
+        tracer.print_computation_log(aggregate, max_depth)
 
     def generate_performance_graph(self, tracer):
         tracer.generate_performance_graph('.')

openfisca_core/tracers/computation_log.py

@@ -30,74 +30,93 @@ def display(
 
         return numpy.array2string(value, max_line_width = float("inf"))
 
-    def _get_node_log(
+    def lines(
             self,
-            node: tracers.TraceNode,
-            depth: int,
-            aggregate: bool,
+            aggregate: bool = False,
+            max_depth: Optional[int] = None,
             ) -> List[str]:
+        depth = 1
+
+        lines_by_tree = [
+            self._get_node_log(node, depth, aggregate, max_depth)
+            for node
+            in self._full_tracer.trees
+            ]
 
-        def print_line(depth: int, node: tracers.TraceNode) -> str:
-            indent = '  ' * depth
-            value = node.value
+        return self._flatten(lines_by_tree)
 
-            if value is None:
-                formatted_value = "{'avg': '?', 'max': '?', 'min': '?'}"
+    def print_log(self, aggregate = False, max_depth = None) -> None:
+        """
+        Print the computation log of a simulation.
 
-            elif aggregate:
-                try:
-                    formatted_value = str({
-                        'avg': numpy.mean(value),
-                        'max': numpy.max(value),
-                        'min': numpy.min(value),
-                        })
+        If ``aggregate`` is ``False`` (default), print the value of each
+        computed vector.
+
+        If ``aggregate`` is ``True``, only print the minimum, maximum, and
+        average value of each computed vector.
+
+        This mode is more suited for simulations on a large population.
 
-                except TypeError:
-                    formatted_value = "{'avg': '?', 'max': '?', 'min': '?'}"
+        If ``max_depth`` is ``None`` (default), print the entire computation.
 
-            else:
-                formatted_value = self.display(value)
+        If ``max_depth`` is set, for example to ``3``, only print computed
+        vectors up to a depth of ``max_depth``.
+        """
+        for line in self.lines(aggregate, max_depth):
+            print(line)  # noqa T001
+
+    def _get_node_log(
+            self,
+            node: tracers.TraceNode,
+            depth: int,
+            aggregate: bool,
+            max_depth: Optional[int],
+            ) -> List[str]:
 
-            return f"{indent}{node.name}<{node.period}> >> {formatted_value}"
+        if max_depth is not None and depth > max_depth:
+            return []
 
-        node_log = [print_line(depth, node)]
+        node_log = [self._print_line(depth, node, aggregate, max_depth)]
 
         children_logs = [
-            self._get_node_log(child, depth + 1, aggregate)
+            self._get_node_log(child, depth + 1, aggregate, max_depth)
             for child
             in node.children
             ]
 
         return node_log + self._flatten(children_logs)
 
-    def _flatten(
+    def _print_line(
             self,
-            list_of_lists: List[List[str]],
-            ) -> List[str]:
-        return [item for _list in list_of_lists for item in _list]
-
-    def lines(self, aggregate: bool = False) -> List[str]:
-        depth = 1
+            depth: int,
+            node: tracers.TraceNode,
+            aggregate: bool,
+            max_depth: Optional[int],
+            ) -> str:
+        indent = '  ' * depth
+        value = node.value
 
-        lines_by_tree = [
-            self._get_node_log(node, depth, aggregate)
-            for node
-            in self._full_tracer.trees
-            ]
+        if value is None:
+            formatted_value = "{'avg': '?', 'max': '?', 'min': '?'}"
 
-        return self._flatten(lines_by_tree)
+        elif aggregate:
+            try:
+                formatted_value = str({
+                    'avg': numpy.mean(value),
+                    'max': numpy.max(value),
+                    'min': numpy.min(value),
+                    })
 
-    def print_log(self, aggregate = False) -> None:
-        """
-        Print the computation log of a simulation.
+            except TypeError:
+                formatted_value = "{'avg': '?', 'max': '?', 'min': '?'}"
 
-        If ``aggregate`` is ``False`` (default), print the value of each
-        computed vector.
+        else:
+            formatted_value = self.display(value)
 
-        If ``aggregate`` is ``True``, only print the minimum, maximum, and
-        average value of each computed vector.
+        return f"{indent}{node.name}<{node.period}> >> {formatted_value}"
 
-        This mode is more suited for simulations on a large population.
-        """
-        for line in self.lines(aggregate):
-            print(line)  # noqa T001
+    def _flatten(
+            self,
+            lists: List[List[str]],
+            ) -> List[str]:
+        return [item for list_ in lists for item in list_]

openfisca_core/tracers/full_tracer.py

@@ -121,8 +121,8 @@ def flat_trace(self) -> tracers.FlatTrace:
     def _get_time_in_sec(self) -> float:
         return time.time_ns() / (10**9)
 
-    def print_computation_log(self, aggregate = False):
-        self.computation_log.print_log(aggregate)
+    def print_computation_log(self, aggregate = False, max_depth = None):
+        self.computation_log.print_log(aggregate, max_depth)
 
     def generate_performance_graph(self, dir_path: str) -> None:
         self.performance_log.generate_graph(dir_path)

setup.py

@@ -48,7 +48,7 @@
 
 setup(
     name = 'OpenFisca-Core',
-    version = '35.8.6',
+    version = '35.9.0',
     author = 'OpenFisca Team',
     author_email = 'contact@openfisca.org',
     classifiers = [

tests/core/test_tracers.py

@@ -421,6 +421,25 @@ def test_log_aggregate_with_strings(tracer):
     assert lines[0] == "  A<2017> >> {'avg': '?', 'max': '?', 'min': '?'}"
 
 
+def test_log_max_depth(tracer):
+    tracer._enter_calculation("A", 2017)
+    tracer._enter_calculation("B", 2017)
+    tracer._enter_calculation("C", 2017)
+    tracer.record_calculation_result(np.asarray([3]))
+    tracer._exit_calculation()
+    tracer.record_calculation_result(np.asarray([2]))
+    tracer._exit_calculation()
+    tracer.record_calculation_result(np.asarray([1]))
+    tracer._exit_calculation()
+
+    assert len(tracer.computation_log.lines()) == 3
+    assert len(tracer.computation_log.lines(max_depth = 4)) == 3
+    assert len(tracer.computation_log.lines(max_depth = 3)) == 3
+    assert len(tracer.computation_log.lines(max_depth = 2)) == 2
+    assert len(tracer.computation_log.lines(max_depth = 1)) == 1
+    assert len(tracer.computation_log.lines(max_depth = 0)) == 0
+
+
 def test_no_wrapping(tracer):
     tracer._enter_calculation("A", 2017)
     tracer.record_calculation_result(HousingOccupancyStatus.encode(np.repeat('tenant', 100)))