Fix unicode conversion bugs and add self-validate pydantic test

Author: bogdandm

json_to_models/generator.py

@@ -1,8 +1,6 @@
 import re
 from typing import Any, Callable, List, Optional, Pattern, Union
 
-from unidecode import unidecode
-
 from .dynamic_typing import (
     ComplexType,
     DDict,
@@ -58,12 +56,8 @@ def _convert(self, data: dict):
         """
         fields = dict()
         for key, value in data.items():
-            # TODO: Check if is 0xC0000005 crash has a place at linux systems
-            # ! _detect_type function can crash at some complex data sets if value is unicode with some characters (maybe German)
-            #   Crash does not produce any useful logs and can occur any time after bad string was processed
-            #   It can be reproduced on real_apis tests (openlibrary API)
             convert_dict = key not in self.dict_keys_fields
-            fields[key] = self._detect_type(value if not isinstance(value, str) else unidecode(value), convert_dict)
+            fields[key] = self._detect_type(value, convert_dict)
         return fields
 
     def _detect_type(self, value, convert_dict=True) -> MetaData:

json_to_models/models/base.py

@@ -220,6 +220,7 @@ def _generate_code(
     """
     imports = []
     classes = []
+    generators = []
     for data in structure:
         nested_imports, nested_classes = _generate_code(
             data["nested"],
@@ -228,7 +229,11 @@ def _generate_code(
             lvl=lvl + 1
         )
         imports.extend(nested_imports)
-        gen = class_generator(data["model"], **class_generator_kwargs)
+        generators.append((
+            class_generator(data["model"], **class_generator_kwargs),
+            nested_classes
+        ))
+    for gen, nested_classes in generators:
         cls_imports, cls_string = gen.generate(nested_classes)
         imports.extend(cls_imports)
         classes.append(cls_string)

test/test_cli/test_self_validate_pydantic.py

@@ -0,0 +1,56 @@
+import imp
+import json
+import sys
+from inspect import isclass
+
+import pydantic
+import pytest
+
+from json_to_models.generator import MetadataGenerator
+from json_to_models.models.base import generate_code
+from json_to_models.models.pydantic import PydanticModelCodeGenerator
+from json_to_models.models.structure import compose_models_flat
+from json_to_models.registry import ModelRegistry
+from .test_script import test_data_path
+
+test_self_validate_pydantic_data = [
+    pytest.param(test_data_path / "gists.json", list, id="gists.json"),
+    pytest.param(test_data_path / "users.json", list, id="users.json"),
+    pytest.param(test_data_path / "unicode.json", dict, id="unicode.json"),
+    pytest.param(test_data_path / "photos.json", dict, id="photos.json"),
+]
+
+
+@pytest.mark.parametrize("data,data_type", test_self_validate_pydantic_data)
+def test_self_validate_pydantic(data, data_type):
+    with data.open() as f:
+        data = json.load(f)
+
+    gen = MetadataGenerator(
+        dict_keys_fields=['files']
+    )
+    reg = ModelRegistry()
+    if data_type is not list:
+        data = [data]
+    fields = gen.generate(*data)
+    reg.process_meta_data(fields, model_name="TestModel")
+    reg.merge_models(generator=gen)
+    reg.generate_names()
+
+    structure = compose_models_flat(reg.models_map)
+    code = generate_code(structure, PydanticModelCodeGenerator)
+    module = imp.new_module("test_models")
+    sys.modules["test_models"] = module
+    try:
+        exec(compile(code, "test_models.py", "exec"), module.__dict__)
+    except Exception as e:
+        assert not e, code
+
+    import test_models
+    for name in dir(test_models):
+        cls = getattr(test_models, name)
+        if isclass(cls) and issubclass(cls, pydantic.BaseModel):
+            cls.update_forward_refs()
+    for item in data:
+        obj = test_models.TestModel.parse_obj(item)
+        assert obj