Handle missing JSON keys, refs #13

Author: simonw

csv_diff/__init__.py

@@ -29,20 +29,26 @@ def load_csv(fp, key=None, dialect=None):
 def load_json(fp, key=None):
     raw_list = json.load(fp)
     assert isinstance(raw_list, list)
+    common_keys = set()
+    for item in raw_list:
+        common_keys.update(item.keys())
     if key:
         keyfn = lambda r: r[key]
     else:
         keyfn = lambda r: hashlib.sha1(
             json.dumps(r, sort_keys=True).encode("utf8")
         ).hexdigest()
-    return {keyfn(r): _simplify_json_row(r) for r in raw_list}
+    return {keyfn(r): _simplify_json_row(r, common_keys) for r in raw_list}
 
 
-def _simplify_json_row(r):
+def _simplify_json_row(r, common_keys):
     # Convert list/dict values into JSON serialized strings
     for key, value in r.items():
         if isinstance(value, (dict, tuple, list)):
             r[key] = json.dumps(value)
+    for key in common_keys:
+        if key not in r:
+            r[key] = None
     return r
 
 

tests/test_cli.py

@@ -23,7 +23,7 @@ def json_files(tmpdir):
     one.write(
         json.dumps(
             [
-                {"id": 1, "name": "Cleo", "nested": {"foo": 3}},
+                {"id": 1, "name": "Cleo", "nested": {"foo": 3}, "extra": 1},
                 {"id": 2, "name": "Pancakes", "nested": {"foo": 3}},
             ]
         )
@@ -32,8 +32,8 @@ def json_files(tmpdir):
     two.write(
         json.dumps(
             [
-                {"id": 1, "name": "Cleo", "nested": {"foo": 3, "bar": 5}},
-                {"id": 2, "name": "Pancakes!", "nested": {"foo": 3}},
+                {"id": 1, "name": "Cleo", "nested": {"foo": 3, "bar": 5}, "extra": 1},
+                {"id": 2, "name": "Pancakes!", "nested": {"foo": 3}, "extra": 1},
             ]
         )
     )
@@ -139,7 +139,10 @@ def test_json_files(json_files):
         "removed": [],
         "changed": [
             {"key": 1, "changes": {"nested": ['{"foo": 3}', '{"foo": 3, "bar": 5}']}},
-            {"key": 2, "changes": {"name": ["Pancakes", "Pancakes!"]}},
+            {
+                "key": 2,
+                "changes": {"name": ["Pancakes", "Pancakes!"], "extra": [None, 1]},
+            },
         ],
         "columns_added": [],
         "columns_removed": [],