Re-introduce array container models (#1503)

* Revert "Add back array models for compatibility (#1499)"

This reverts commit 1c813ad.

* re-add old models

* generate

* cleanup

* add back list models to apis

* fix naming

* fix typing

* fix more models and typing info

* remove models that didnt exist in 2.12.0

* docs

* cleanup generator

* update examples

* fix

* fix Simple model unparsed

* undo models added after 2.12.0

Author: skarimo

.generator/conftest.py

@@ -17,7 +17,7 @@
 
 from generator import openapi
 
-from generator.formatter import format_parameters, format_data_with_schema, safe_snake_case, snake_case
+from generator.formatter import format_parameters, format_data_with_schema, safe_snake_case, snake_case, set_api_version
 
 
 MODIFIED_FEATURES = {
@@ -156,6 +156,7 @@ def code_examples():
 @pytest.fixture
 def api_version(request):
     path = pathlib.Path(request.node.__scenario_report__.scenario.feature.filename)
+    set_api_version(path.parent.parent.name)
     return path.parent.parent.name
 
 

.generator/src/generator/cli.py

@@ -64,7 +64,6 @@ def cli(specs, output):
     models_j2 = env.get_template("models.j2")
     init_j2 = env.get_template("init.j2")
     configuration_j2 = env.get_template("configuration.j2")
-    compat_j2 = env.get_template("compat.j2")
 
     extra_files = {
         "api_client.py": env.get_template("api_client.j2"),
@@ -73,6 +72,7 @@ def cli(specs, output):
         "rest.py": env.get_template("rest.j2"),
     }
 
+
     top_package = output / PACKAGE_NAME
     top_package.mkdir(parents=True, exist_ok=True)
 
@@ -90,6 +90,7 @@ def cli(specs, output):
 
         version = spec_path.parent.name
         env.globals["version"] = version
+        formatter.set_api_version(version)
 
         all_specs[version] = spec
 
@@ -141,9 +142,3 @@ def cli(specs, output):
     filename = top_package / "configuration.py"
     with filename.open("w") as fp:
         fp.write(configuration_j2.render(specs=all_specs, apis=all_apis))
-
-    with (pathlib.Path(__file__).parent / "compat-files").open() as fp:
-        for compat_file in fp:
-            compat_file, compat_model = compat_file.strip().split(":")
-            with top_package.joinpath(compat_file).open("w") as model_fp:
-                model_fp.write(compat_j2.render(model=compat_model))

.generator/src/generator/compat-files

@@ -20,6 +20,52 @@
     with replacement_file.open() as f:
         EDGE_CASES.update(json.load(f))
 
+API_VERSION = None
+WHITELISTED_LIST_MODELS={
+    "v1": (
+        "AgentCheck",
+        "AzureAccountListResponse",
+        "DashboardBulkActionDataList",
+        "DistributionPoint",
+        "DistributionPointData",
+        "GCPAccountListResponse",
+        "HTTPLog",
+        "LogsPipelineList",
+        "MonitorSearchCount",
+        "Point",
+        "ServiceChecks",
+        "SharedDashboardInvitesDataList",
+        "SlackIntegrationChannels",
+        "SyntheticsRestrictedRoles",
+        "UsageAttributionAggregates",
+    ),
+    "v2": (
+        "CIAppAggregateBucketValueTimeseries",
+        "EventsQueryGroupBys",
+        "GroupTags",
+        "HTTPLog",
+        "IncidentTodoAssigneeArray",
+        "LogsAggregateBucketValueTimeseries",
+        "MetricBulkTagConfigEmailList",
+        "MetricBulkTagConfigTagNameList",
+        "MetricCustomAggregations",
+        "MetricSuggestedAggregations",
+        "RUMAggregateBucketValueTimeseries",
+        "ScalarFormulaRequestQueries",
+        "SecurityMonitoringSignalIncidentIds",
+        "SensitiveDataScannerGetConfigIncludedArray",
+        "SensitiveDataScannerStandardPatternsResponse",
+        "TagsEventAttribute",
+        "TeamPermissionSettingValues",
+        "TimeseriesFormulaRequestQueries",
+        "TimeseriesResponseSeriesList",
+        "TimeseriesResponseTimes",
+        "TimeseriesResponseValues",
+        "TimeseriesResponseValuesList",
+
+    ),
+}
+
 KEYWORDS = set(keyword.kwlist)
 KEYWORDS.add("property")
 KEYWORDS.add("cls")
@@ -30,6 +76,15 @@
 PATTERN_WHITESPACE = re.compile(r"\W")
 
 
+def set_api_version(version):
+    global API_VERSION
+    API_VERSION = version
+
+
+def is_list_model_whitelisted(name):
+    return name in WHITELISTED_LIST_MODELS[API_VERSION]
+
+
 def snake_case(value):
     s1 = PATTERN_LEADING_ALPHA.sub(r"\1_\2", value)
     s1 = PATTERN_FOLLOWING_ALPHA.sub(r"\1_\2", s1).lower()
@@ -237,7 +292,11 @@ def format_data_with_schema_list(
 ):
     """Format data with schema."""
     assert version is not None
+
     imports = imports or defaultdict(set)
+    name, r_imports = get_name_and_imports(schema, version, None)
+    if is_list_model_whitelisted(name):
+        imports.update(r_imports) 
 
     if "oneOf" in schema:
         for sub_schema in schema["oneOf"]:
@@ -246,6 +305,7 @@ def format_data_with_schema_list(
                     data,
                     sub_schema,
                     replace_values=replace_values,
+                    default_name=name if is_list_model_whitelisted(name) else None,
                     version=version,
                 )
             except (KeyError, ValueError):
@@ -269,6 +329,9 @@ def format_data_with_schema_list(
         imports = _merge_imports(imports, extra_imports)
     parameters = f"[{parameters}]"
 
+    if name and is_list_model_whitelisted(name): 
+        return f"{name}({parameters})", imports
+
     return parameters, imports
 
 

.generator/src/generator/formatter.py

@@ -10,6 +10,7 @@
 
 from . import formatter
 
+
 PRIMITIVE_TYPES = ["string", "number", "boolean", "integer"]
 
 
@@ -24,7 +25,6 @@ def type_to_python_helper(type_, schema, alternative_name=None, in_list=False, t
         if typing:
             return "Any"
         return "bool, date, datetime, dict, float, int, list, str, none_type"
-
     if type_ == "integer":
         return "int"
     elif type_ == "number":
@@ -71,11 +71,13 @@ def type_to_python_helper(type_, schema, alternative_name=None, in_list=False, t
 
 def type_to_python(schema, alternative_name=None, in_list=False, typing=False):
     """Return Python type name for the type."""
+
     name = formatter.get_name(schema)
-    if name and "items" not in schema:
+    # TODO: double check
+    if name and "items" not in schema or formatter.is_list_model_whitelisted(name):
         if "enum" in schema:
             return name
-        if schema.get("type", "object") == "object":
+        if schema.get("type", "object") == "object" or formatter.is_list_model_whitelisted(name):
             if typing and "oneOf" in schema:
                 types = [name]
                 types.extend(get_oneof_types(schema, typing=typing))
@@ -168,7 +170,7 @@ def child_models(schema, alternative_name=None, seen=None, in_list=False):
 
     has_sub_models = False
     if "oneOf" in schema:
-        has_sub_models = not in_list
+        has_sub_models = not in_list and not formatter.is_list_model_whitelisted(name)
         for child in schema["oneOf"]:
             sub_models = list(child_models(child, seen=seen))
             if sub_models:
@@ -178,7 +180,12 @@ def child_models(schema, alternative_name=None, seen=None, in_list=False):
             return
 
     if "items" in schema:
-        yield from child_models(schema["items"], alternative_name=name + "Item" if name is not None else None, seen=seen, in_list=True)
+        if formatter.is_list_model_whitelisted(name):
+            alt_name = None
+        else:
+            alt_name = name + "Item" if name is not None else None
+
+        yield from child_models(schema["items"], alternative_name=alt_name, seen=seen, in_list=True)
 
     if schema.get("type") == "object" or "properties" in schema or has_sub_models:
         if not has_sub_models and name is None:
@@ -202,6 +209,15 @@ def child_models(schema, alternative_name=None, seen=None, in_list=False):
         for key, child in schema.get("properties", {}).items():
             yield from child_models(child, alternative_name=name + formatter.camel_case(key), seen=seen)
 
+    if current_name and schema.get("type") == "array":
+        if name in seen:
+            return
+
+        if formatter.is_list_model_whitelisted(name):
+            seen.add(name)
+            yield name, schema
+
+
     if "enum" in schema:
         if name is None:
             raise ValueError(f"Schema {schema} has no name")
@@ -271,11 +287,17 @@ def get_references_for_model(model, model_name):
                 if name:
                     result[name] = None
         elif definition.get("type") == "array":
-            name = formatter.get_name(definition.get("items"))
-            if name and find_non_primitive_type(definition["items"]):
+            name = formatter.get_name(definition)
+            if name and formatter.is_list_model_whitelisted(name):
                 result[name] = None
-            elif formatter.get_name(definition) and definition["items"].get("type") not in PRIMITIVE_TYPES:
-                result[formatter.get_name(definition) + "Item"] = None
+            else:
+                name = formatter.get_name(definition.get("items"))
+                if name and find_non_primitive_type(definition["items"]):
+                    result[name] = None
+                elif name and formatter.is_list_model_whitelisted(name):
+                    result[name] = None
+                elif formatter.get_name(definition) and definition["items"].get("type") not in PRIMITIVE_TYPES:
+                    result[formatter.get_name(definition) + "Item"] = None
 
         elif definition.get("properties") and top_name:
             result[top_name + formatter.camel_case(key)] = None
@@ -305,8 +327,10 @@ def get_oneof_references_for_model(model, model_name, seen=None):
     if model.get("oneOf"):
         for schema in model["oneOf"]:
             type_ = schema.get("type", "object")
-            if type_ == "object":
-                result[formatter.get_name(schema)] = None
+
+            oneof_name = formatter.get_name(schema)
+            if type_ == "object" or formatter.is_list_model_whitelisted(oneof_name):
+                result[oneof_name] = None
             elif type_ == "array":
                 sub_name = formatter.get_name(schema["items"])
                 if sub_name:
@@ -336,8 +360,9 @@ def get_oneof_parameters(model):
 def get_oneof_types(model, typing=False):
     for schema in model["oneOf"]:
         type_ = schema.get("type", "object")
-        if type_ == "object":
-            yield formatter.get_name(schema)
+        name = formatter.get_name(schema)
+        if type_ == "object" or formatter.is_list_model_whitelisted(name):
+            yield name
         elif type_ == "array":
             name = formatter.get_name(schema["items"])
             if name:
@@ -361,8 +386,9 @@ def get_oneof_models(model):
     result = []
     for schema in model["oneOf"]:
         type_ = schema.get("type", "object")
-        if type_ == "object":
-            result.append(formatter.get_name(schema))
+        name = formatter.get_name(schema)
+        if type_ == "object" or formatter.is_list_model_whitelisted(name):
+            result.append(name)
         elif type_ == "array":
             name = formatter.get_name(schema["items"])
             if name:

.generator/src/generator/openapi.py

@@ -1527,9 +1527,10 @@ def get_oneof_instance(cls, model_kwargs, constant_kwargs, model_arg=None):
                     if list_oneof_instance:
                         oneof_instances.append(list_oneof_instance)
                 elif issubclass(oneof_class, ModelSimple):
-                    oneof_instance = oneof_class(model_arg, **constant_kwargs)
-                    if not oneof_instance._unparsed:
-                        oneof_instances.append(oneof_instance)
+                    if model_arg is not None:
+                        oneof_instance = oneof_class(model_arg, **constant_kwargs)
+                        if not oneof_instance._unparsed:
+                            oneof_instances.append(oneof_instance)
                 elif oneof_class in PRIMITIVE_TYPES:
                     oneof_instance = validate_and_convert_types(
                         model_arg,

.generator/src/generator/templates/compat.j2

@@ -2052,13 +2052,6 @@ monitor\_search\_count
    :members:
    :show-inheritance:
 
-monitor\_search\_count\_item
-----------------------------
-
-.. automodule:: datadog_api_client.v1.model.monitor_search_count_item
-   :members:
-   :show-inheritance:
-
 monitor\_search\_response
 -------------------------
 
@@ -2500,27 +2493,13 @@ notify\_end\_state
    :members:
    :show-inheritance:
 
-notify\_end\_states
--------------------
-
-.. automodule:: datadog_api_client.v1.model.notify_end_states
-   :members:
-   :show-inheritance:
-
 notify\_end\_type
 -----------------
 
 .. automodule:: datadog_api_client.v1.model.notify_end_type
    :members:
    :show-inheritance:
 
-notify\_end\_types
-------------------
-
-.. automodule:: datadog_api_client.v1.model.notify_end_types
-   :members:
-   :show-inheritance:
-
 on\_missing\_data\_option
 -------------------------