Handle columns being added and removed

My sf-tree-history project unexpectedly broke because the columns
in the CSV file changed.

https://circleci.com/gh/simonw/sf-tree-history/110

csv-diff can now detect and handle this without crashing.

Author: simonw

README.md

@@ -71,7 +71,9 @@ You can also run it using the `--json` option to get a machine-readable differen
                     ]
                 }
             }
-        ]
+        ],
+        "columns_added": [],
+        "columns_removed": []
     }
 
 You can also import the Python library into your own code like so:
@@ -83,3 +85,5 @@ You can also import the Python library into your own code like so:
     )
 
 `diff` will now contain the same data structure as the output in the `--json` example above.
+
+If the columns in the CSV have changed, those added or removed olumns will be ignored when calculating changes made to specific rows.

csv_diff/__init__.py

@@ -16,30 +16,49 @@ def load_csv(fp, key=None):
 
 
 def compare(previous, current):
+    result = {
+        "added": [],
+        "removed": [],
+        "changed": [],
+        "columns_added": [],
+        "columns_removed": [],
+    }
+    # Have the columns changed?
+    previous_columns = set(next(iter(previous.values())).keys())
+    current_columns = set(next(iter(current.values())).keys())
+    ignore_columns = None
+    if previous_columns != current_columns:
+        result["columns_added"] = [
+            c for c in current_columns if c not in previous_columns
+        ]
+        result["columns_removed"] = [
+            c for c in previous_columns if c not in current_columns
+        ]
+        ignore_columns = current_columns.symmetric_difference(previous_columns)
     # Have any rows been removed or added?
     removed = [id for id in previous if id not in current]
     added = [id for id in current if id not in previous]
     # How about changed?
     removed_or_added = set(removed) | set(added)
     potential_changes = [id for id in current if id not in removed_or_added]
     changed = [id for id in potential_changes if current[id] != previous[id]]
-    result = {"added": [], "removed": [], "changed": []}
     if added:
         result["added"] = [current[id] for id in added]
     if removed:
         result["removed"] = [previous[id] for id in removed]
     if changed:
         for id in changed:
-            d = list(diff(previous[id], current[id]))
-            result["changed"].append(
-                {
-                    "key": id,
-                    "changes": {
-                        field: [prev_value, current_value]
-                        for _, field, (prev_value, current_value) in d
-                    },
-                }
-            )
+            d = list(diff(previous[id], current[id], ignore=ignore_columns))
+            if d:
+                result["changed"].append(
+                    {
+                        "key": id,
+                        "changes": {
+                            field: [prev_value, current_value]
+                            for _, field, (prev_value, current_value) in d
+                        },
+                    }
+                )
     return result
 
 
@@ -49,6 +68,28 @@ def human_text(result, key=None, singular=None, plural=None):
     title = []
     summary = []
     show_headers = sum(1 for key in result if result[key]) > 1
+    if result["columns_added"]:
+        fragment = "{} {} added".format(
+            len(result["columns_added"]),
+            "column" if len(result["columns_added"]) == 1 else "columns",
+        )
+        title.append(fragment)
+        summary.extend(
+            [fragment]
+            + ["    {}".format(c) for c in sorted(result["columns_added"])]
+            + [""]
+        )
+    if result["columns_removed"]:
+        fragment = "{} {} removed".format(
+            len(result["columns_removed"]),
+            "column" if len(result["columns_removed"]) == 1 else "columns",
+        )
+        title.append(fragment)
+        summary.extend(
+            [fragment]
+            + ["    {}".format(c) for c in sorted(result["columns_removed"])]
+            + [""]
+        )
     if result["changed"]:
         fragment = "{} {} changed".format(
             len(result["changed"]), singular if len(result["changed"]) == 1 else plural

tests/test_cli.py

@@ -71,4 +71,6 @@ def test_human_cli_json(tmpdir):
         "added": [],
         "removed": [],
         "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
+        "columns_added": [],
+        "columns_removed": [],
     } == json.loads(result.output.strip())

tests/test_csv_diff.py

@@ -27,6 +27,14 @@
 1,Cleo,5
 3,Bailey,1"""
 
+SEVEN = """id,name,weight
+1,Cleo,48
+3,Bailey,20"""
+
+EIGHT = """id,name,age,length
+3,Bailee,1,100
+4,Bob,7,422"""
+
 
 def test_row_changed():
     diff = compare(
@@ -36,6 +44,8 @@ def test_row_changed():
         "added": [],
         "removed": [],
         "changed": [{"key": "1", "changes": {"age": ["4", "5"]}}],
+        "columns_added": [],
+        "columns_removed": [],
     } == diff
 
 
@@ -47,6 +57,8 @@ def test_row_added():
         "changed": [],
         "removed": [],
         "added": [{"age": "2", "id": "2", "name": "Pancakes"}],
+        "columns_added": [],
+        "columns_removed": [],
     } == diff
 
 
@@ -58,4 +70,19 @@ def test_row_removed():
         "changed": [],
         "removed": [{"age": "2", "id": "2", "name": "Pancakes"}],
         "added": [],
+        "columns_added": [],
+        "columns_removed": [],
+    } == diff
+
+
+def test_columns_changed():
+    diff = compare(
+        load_csv(io.StringIO(SIX), key="id"), load_csv(io.StringIO(SEVEN), key="id")
+    )
+    assert {
+        "changed": [],
+        "removed": [],
+        "added": [],
+        "columns_added": ["weight"],
+        "columns_removed": ["age"],
     } == diff

tests/test_human_text.py

@@ -1,5 +1,5 @@
 from csv_diff import load_csv, compare, human_text
-from .test_csv_diff import ONE, TWO, THREE, FOUR, FIVE, SIX
+from .test_csv_diff import ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT
 from textwrap import dedent
 import io
 
@@ -113,3 +113,62 @@ def test_row_changed_and_row_added_and_row_deleted():
         ).strip()
         == human_text(diff, "id")
     )
+
+
+def test_columns_changed():
+    diff = compare(
+        load_csv(io.StringIO(SIX), key="id"), load_csv(io.StringIO(SEVEN), key="id")
+    )
+    assert (
+        dedent(
+            """
+    1 column added, 1 column removed
+
+    1 column added
+        weight
+
+    1 column removed
+        age
+    """
+        ).strip()
+        == human_text(diff, "id")
+    )
+
+
+def test_columns_and_rows_changed():
+    diff = compare(
+        load_csv(io.StringIO(SEVEN), key="id"), load_csv(io.StringIO(EIGHT), key="id")
+    )
+    assert (
+        dedent(
+            """
+    2 columns added, 1 column removed, 1 row changed, 1 row added, 1 row removed
+
+    2 columns added
+        age
+        length
+
+    1 column removed
+        weight
+
+    1 row changed
+
+      id: 3
+        name: "Bailey" => "Bailee"
+
+    1 row added
+
+      id: 4
+      name: Bob
+      age: 7
+      length: 422
+
+    1 row removed
+
+      id: 1
+      name: Cleo
+      weight: 48
+    """
+        ).strip()
+        == human_text(diff, "id")
+    )