Merge pull request #25 from bogdandm/swagger-json-test

Bug fixes

Author: bogdandm

README.md

@@ -40,6 +40,11 @@ json2python-models is a [Python](https://www.python.org/) tool that can generate
 
 ## Example
 
+### F1 Season Results
+
+<details><summary>Show (long code)</summary>
+<p>
+
 ```
 driver_standings.json
 [
@@ -121,6 +126,142 @@ class DriverStandings:
     driver_standings: List['DriverStanding'] = attr.ib()
 ```
 
+</p>
+</details>
+
+### Swagger
+
+<details><summary>Show (long code)</summary>
+<p>
+
+`swagger.json` from any online API (I tested file generated by drf-yasg and another one for Spotify API)
+
+It requires a lit bit of tweaking:
+* Some fields store routes/models specs as dicts
+* There is a lot of optinal fields so we reduce merging threshold 
+
+```
+json_to_models -s flat -f dataclasses -m Swagger testing_tools/swagger.json 
+    --dict-keys-fields securityDefinitions paths responses definitions properties 
+    --merge percent_50 number
+```
+
+```python
+from dataclasses import dataclass, field
+from json_to_models.dynamic_typing import FloatString
+from typing import Any, Dict, List, Optional, Union
+
+
+@dataclass
+class Swagger:
+    swagger: FloatString
+    info: 'Info'
+    host: str
+    schemes: List[str]
+    base_path: str
+    consumes: List[str]
+    produces: List[str]
+    security_definitions: Dict[str, 'Parameter_SecurityDefinition']
+    security: List['Security']
+    paths: Dict[str, 'Path']
+    definitions: Dict[str, 'Definition_Schema']
+
+
+@dataclass
+class Info:
+    title: str
+    description: str
+    version: str
+
+
+@dataclass
+class Security:
+    api_key: Optional[List[Any]] = field(default_factory=list)
+    basic: Optional[List[Any]] = field(default_factory=list)
+
+
+@dataclass
+class Path:
+    parameters: List['Parameter_SecurityDefinition']
+    post: Optional['Delete_Get_Patch_Post_Put'] = None
+    get: Optional['Delete_Get_Patch_Post_Put'] = None
+    put: Optional['Delete_Get_Patch_Post_Put'] = None
+    patch: Optional['Delete_Get_Patch_Post_Put'] = None
+    delete: Optional['Delete_Get_Patch_Post_Put'] = None
+
+
+@dataclass
+class Property:
+    type: str
+    format: Optional[str] = None
+    xnullable: Optional[bool] = None
+    items: Optional['Item_Schema'] = None
+
+
+@dataclass
+class Property_2E:
+    type: str
+    title: Optional[str] = None
+    read_only: Optional[bool] = None
+    max_length: Optional[int] = None
+    min_length: Optional[int] = None
+    items: Optional['Item'] = None
+    enum: Optional[List[str]] = field(default_factory=list)
+    maximum: Optional[int] = None
+    minimum: Optional[int] = None
+    format: Optional[str] = None
+
+
+@dataclass
+class Item:
+    ref: Optional[str] = None
+    title: Optional[str] = None
+    type: Optional[str] = None
+    max_length: Optional[int] = None
+    min_length: Optional[int] = None
+
+
+@dataclass
+class Parameter_SecurityDefinition:
+    name: str
+    in_: str
+    required: Optional[bool] = None
+    schema: Optional['Item_Schema'] = None
+    type: Optional[str] = None
+    description: Optional[str] = None
+
+
+@dataclass
+class Delete_Get_Patch_Post_Put:
+    operation_id: str
+    description: str
+    parameters: List['Parameter_SecurityDefinition']
+    responses: Dict[str, 'Response']
+    tags: List[str]
+
+
+@dataclass
+class Item_Schema:
+    ref: str
+
+
+@dataclass
+class Response:
+    description: str
+    schema: Optional[Union['Item_Schema', 'Definition_Schema']] = None
+
+
+@dataclass
+class Definition_Schema:
+    ref: Optional[str] = None
+    required: Optional[List[str]] = field(default_factory=list)
+    type: Optional[str] = None
+    properties: Optional[Dict[str, Union['Property_2E', 'Property']]] = field(default_factory=dict)
+```
+
+</p>
+</details>
+
 ## Installation
 
 | **Be ware**: this project supports only `python3.7` and higher. |

json_to_models/generator.py

@@ -237,23 +237,24 @@ def _optimize_union(self, t: DUnion):
 
         types = [self.optimize_type(t) for t in other_types]
 
-        if Unknown in types:
-            types.remove(Unknown)
-
-        optional = False
-        if Null in types:
-            optional = True
-            while Null in types:
-                types.remove(Null)
 
         if len(types) > 1:
+            if Unknown in types:
+                types.remove(Unknown)
+
+            optional = False
+            if Null in types:
+                optional = True
+                while Null in types:
+                    types.remove(Null)
+
             meta_type = DUnion(*types)
             if len(meta_type.types) == 1:
                 meta_type = meta_type.types[0]
+
+            if optional:
+                return DOptional(meta_type)
         else:
             meta_type = types[0]
 
-        if optional:
-            return DOptional(meta_type)
-        else:
-            return meta_type
+        return meta_type

json_to_models/models/base.py

@@ -75,7 +75,7 @@ def __init__(self, model: ModelMeta, post_init_converters=False, convert_unicode
         self.model = model
         self.post_init_converters = post_init_converters
         self.convert_unicode = convert_unicode
-        self.model.name = self.convert_class_name(self.model.name)
+        self.model.set_raw_name(self.convert_class_name(self.model.name), generated=self.model.is_name_generated)
 
     @cached_method
     def convert_class_name(self, name):

json_to_models/registry.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 from itertools import chain, combinations
-from typing import Dict, Iterable, List, Set, Tuple
+from typing import Dict, FrozenSet, Iterable, List, Set, Tuple
 
 from ordered_set import OrderedSet
 
@@ -153,15 +153,22 @@ def merge_models(self, generator, strict=False) -> List[Tuple[ModelMeta, Set[Mod
         flag = True
         while flag:
             flag = False
-            new_groups: OrderedSet[Set[ModelMeta]] = OrderedSet()
-            for gr1, gr2 in combinations(groups, 2):
-                if gr1 & gr2:
-                    old_len = len(new_groups)
-                    new_groups.add(frozenset(gr1 | gr2))
-                    added = old_len < len(new_groups)
-                    flag = flag or added
+            new_groups: OrderedSet[FrozenSet[ModelMeta]] = OrderedSet()
+            for gr1 in groups:
+                in_set = False
+                for gr2 in groups:
+                    if gr1 is gr2:
+                        continue
+                    if gr1 & gr2:
+                        in_set = True
+                        old_len = len(new_groups)
+                        new_groups.add(frozenset(gr1 | gr2))
+                        added = old_len < len(new_groups)
+                        flag = flag or added
+                if not in_set:
+                    new_groups.add(gr1)
             if flag:
-                groups = new_groups
+                groups: OrderedSet[FrozenSet[ModelMeta]] = new_groups
 
         replaces = []
         replaces_ids = set()
@@ -172,8 +179,7 @@ def merge_models(self, generator, strict=False) -> List[Tuple[ModelMeta, Set[Mod
             replaces.append((model_meta, group))
 
         for model_meta in self.models:
-            if model_meta.index not in replaces_ids:
-                generator.optimize_type(model_meta)
+            generator.optimize_type(model_meta)
         return replaces
 
     def _merge(self, generator, *models: ModelMeta):

test/test_registry/test_registry_merge_models.py

@@ -214,6 +214,58 @@
         ],
         id="merge_models_with_optional_field"
     ),
+    pytest.param(
+        [
+            {
+                "a" + str(i): int for i in range(20)
+            },
+            {
+                **{
+                    "a" + str(i): int for i in range(20)
+                },
+                **{
+                    "b" + str(i): int for i in range(20)
+                }
+            },
+            {
+                "b" + str(i): int for i in range(20)
+            },
+            {
+                "c" + str(i): int for i in range(20)
+            },
+            {
+                **{
+                    "b" + str(i): int for i in range(20)
+                },
+                **{
+                    "c" + str(i): int for i in range(20)
+                }
+            },
+            {
+                "field1": int
+            },
+            {
+                "field1": int
+            }
+        ],
+        [
+            {
+                **{
+                    "a" + str(i): DOptional(int) for i in range(20)
+                },
+                **{
+                    "b" + str(i): DOptional(int) for i in range(20)
+                },
+                **{
+                    "c" + str(i): DOptional(int) for i in range(20)
+                }
+            },
+            {
+                "field1": int
+            }
+        ],
+        id="multistage_merge"
+    ),
 ]
 
 

testing_tools/real_apis/spotify-swagger.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+
+import yaml
+
+from json_to_models.dynamic_typing.string_serializable import StringSerializable, registry
+from json_to_models.generator import MetadataGenerator
+from json_to_models.models.attr import AttrsModelCodeGenerator
+from json_to_models.models.base import generate_code
+from json_to_models.models.structure import compose_models_flat
+from json_to_models.registry import ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry
+
+
+@registry.add()
+class SwaggerRef(StringSerializable, str):
+    @classmethod
+    def to_internal_value(cls, value: str) -> 'SwaggerRef':
+        if not value.startswith("#/"):
+            raise ValueError(f"invalid literal for SwaggerRef: '{value}'")
+        return cls(value)
+
+    def to_representation(self) -> str:
+        return str(self)
+
+
+def load_data() -> dict:
+    with (Path(__file__) / ".." / ".." / "spotify-swagger.yaml").open() as f:
+        data = yaml.load(f, Loader=yaml.SafeLoader)
+    return data
+
+
+def main():
+    data = load_data()
+    del data["paths"]
+
+    gen = MetadataGenerator(
+        dict_keys_regex=[],
+        dict_keys_fields=["securityDefinitions", "paths", "responses", "definitions", "properties", "scopes"]
+    )
+    reg = ModelRegistry(ModelFieldsPercentMatch(.5), ModelFieldsNumberMatch(10))
+    fields = gen.generate(data)
+    reg.process_meta_data(fields, model_name="Swagger")
+    reg.merge_models(generator=gen)
+    reg.generate_names()
+
+    structure = compose_models_flat(reg.models_map)
+    code = generate_code(structure, AttrsModelCodeGenerator)
+    print(code)
+
+
+if __name__ == '__main__':
+    main()