sparse codegen

src/aws-cpp-sdk-core/include/aws/core/utils/json/JsonSerializer.h

@@ -94,6 +94,12 @@ namespace Aws
                 JsonValue& WithString(const Aws::String& key, const Aws::String& value);
                 JsonValue& WithString(const char* key, const Aws::String& value);
 
+                /**
+                 * Adds a null value to the top level of this node with key.
+                 */
+                JsonValue& WithNull(const Aws::String& key);
+                JsonValue& WithNull(const char* key);
+
                 /**
                  * Converts the current JSON node to a string.
                  */

src/aws-cpp-sdk-core/source/utils/json/JsonSerializer.cpp

@@ -153,6 +153,23 @@ JsonValue& JsonValue::WithString(const Aws::String& key, const Aws::String& valu
     return WithString(key.c_str(), value);
 }
 
+JsonValue& JsonValue::WithNull(const char* key)
+{
+    if (!m_value)
+    {
+        m_value = cJSON_AS4CPP_CreateObject();
+    }
+
+    const auto val = cJSON_AS4CPP_CreateNull();
+    AddOrReplace(m_value, key, val);
+    return *this;
+}
+
+JsonValue& JsonValue::WithNull(const Aws::String& key)
+{
+    return WithNull(key.c_str());
+}
+
 JsonValue& JsonValue::AsString(const Aws::String& value)
 {
     Destroy();

tests/aws-cpp-sdk-core-tests/utils/json/JsonSerializerTest.cpp

@@ -434,6 +434,39 @@ TEST_F(JsonSerializerTest, TestGetAllObjects)
     ASSERT_EQ(42, all["Key2"].AsInteger());
 }
 
+TEST_F(JsonSerializerTest, TestWithNull)
+{
+    JsonValue value;
+    value.WithString("Key1", "value1");
+    value.WithNull("Key2");
+    value.WithNull(Aws::String("Key3"));
+    value.WithInteger("Key4", 42);
+
+    auto view = value.View();
+
+    ASSERT_TRUE(view.KeyExists("Key2"));
+    ASSERT_FALSE(view.ValueExists("Key2"));
+    ASSERT_TRUE(view.GetObject("Key2").IsNull());
+
+    ASSERT_TRUE(view.KeyExists("Key3"));
+    ASSERT_FALSE(view.ValueExists("Key3"));
+    ASSERT_TRUE(view.GetObject("Key3").IsNull());
+
+    ASSERT_STREQ("value1", view.GetString("Key1").c_str());
+    ASSERT_EQ(42, view.GetInteger("Key4"));
+
+    Aws::String serialized = view.WriteCompact();
+    JsonValue reparsed(serialized);
+    ASSERT_TRUE(reparsed.WasParseSuccessful());
+    auto reparsedView = reparsed.View();
+    ASSERT_TRUE(reparsedView.KeyExists("Key2"));
+    ASSERT_FALSE(reparsedView.ValueExists("Key2"));
+    ASSERT_TRUE(reparsedView.GetObject("Key2").IsNull());
+    ASSERT_TRUE(reparsedView.KeyExists("Key3"));
+    ASSERT_FALSE(reparsedView.ValueExists("Key3"));
+    ASSERT_TRUE(reparsedView.GetObject("Key3").IsNull());
+}
+
 TEST_F(JsonSerializerTest, TestEquality)
 {
     auto input = R"({"AWS" : {

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/c2j/C2jShape.java

@@ -36,4 +36,5 @@ public class C2jShape {
     private boolean sensitive;
     private boolean document;
     private Map<String, Boolean> retryable;
+    private boolean sparse;
 }

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/codegeneration/Shape.java

@@ -63,6 +63,7 @@ public class Shape {
     private boolean overrideStreaming = false;
     private boolean requestCompressionRequired=false;
     private boolean requestCompressionRequiredGzip=false;
+    private boolean sparse=false;
 
     public boolean isMap() {
         return "map".equals(type.toLowerCase());

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/domainmodels/codegeneration/cpp/CppViewHelper.java

@@ -26,6 +26,7 @@
 
 public class CppViewHelper {
     private static final Map<String, String> CORAL_TYPE_TO_CPP_TYPE_MAPPING = new HashMap<>();
+    private static final Map<String, String> CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING = new HashMap<>();
     private static final Map<String, String> CORAL_TYPE_TO_JSON_CPP_TYPE_MAPPING = new HashMap<>();
     private static final Map<String, String> CORAL_TYPE_TO_XML_CONVERSION_MAPPING = new HashMap<>();
     private static final Map<String, String> CORAL_TYPE_TO_DEFAULT_VALUES = new HashMap<>();
@@ -56,6 +57,22 @@ public class CppViewHelper {
         CORAL_TYPE_TO_CPP_TYPE_MAPPING.put("sensitive_double", "double");
         CORAL_TYPE_TO_CPP_TYPE_MAPPING.put("sensitive_timestamp", "Aws::Utils::DateTime");
 
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("long", "long long");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("integer", "int64_t");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("string", "Aws::String");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("timestamp", "Aws::Utils::DateTime");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("boolean", "bool");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_boolean", "bool");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("double", "double");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("float", "double");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("blob", "Aws::Utils::ByteBuffer");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_blob", "Aws::Utils::CryptoBuffer");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_long", "long long");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_integer", "int64_t");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_float", "double");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_double", "double");
+        CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING.put("sensitive_timestamp", "Aws::Utils::DateTime");
+
         CORAL_TYPE_TO_JSON_CPP_TYPE_MAPPING.put("long", "Int64");
         CORAL_TYPE_TO_JSON_CPP_TYPE_MAPPING.put("integer", "Integer");
         CORAL_TYPE_TO_JSON_CPP_TYPE_MAPPING.put("string", "String");
@@ -225,12 +242,14 @@ else if(shape.isStructure() || shape.isEnum())
 
         else if(shape.isList()) {
             String type = computeCppTypeInternal(shape.getListMember().getShape(), typeMapping);
+            type = shape.isSparse() ? String.format("Aws::Crt::Optional<%s>", type) : type;
             return String.format("Aws::Vector<%s>", type);
         }
 
         else if(shape.isMap()) {
             String key = computeCppTypeInternal(shape.getMapKey().getShape(), typeMapping);
             String value = computeCppTypeInternal(shape.getMapValue().getShape(), typeMapping);
+            value = shape.isSparse() ? String.format("Aws::Crt::Optional<%s>", value) : value;
             return String.format("Aws::Map<%s, %s>", key, value);
         }
 
@@ -243,6 +262,16 @@ public static String computeCppType(Shape shape) {
         return computeCppTypeInternal(shape, CORAL_TYPE_TO_CPP_TYPE_MAPPING);
     }
 
+    public static String computeCborCppType(Shape shape) {
+        return computeCppTypeInternal(shape, CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING);
+    }
+
+    public static String computeResultCppType(Shape shape, String protocol) {
+        return "smithy-rpc-v2-cbor".equals(protocol)
+                ? computeCppTypeInternal(shape, CORAL_TYPE_TO_CBOR_CPP_TYPE_MAPPING)
+                : computeCppTypeInternal(shape, CORAL_TYPE_TO_CPP_TYPE_MAPPING);
+    }
+
     public static boolean isStreamingPayloadMember(Shape parent, String member) {
         if (!parent.getMembers().containsKey(member)) {
             throw new RuntimeException("Parent shape " + parent.getName() +
@@ -275,6 +304,22 @@ public static String computeCppType(Shape parent, String member) {
         return computeCppType(childShape);
     }
 
+    public static String computeCborCppType(Shape parent, String member) {
+        if (!parent.getMembers().containsKey(member)) {
+            throw new RuntimeException("Parent shape " + parent.getName() +
+                    " does not contain member key " + member);
+        }
+        ShapeMember shapeMember = parent.getMembers().get(member);
+        Shape childShape = shapeMember.getShape();
+
+        if (parent.getPayload() != null && parent.getPayload().equals(member) && parent.isResult()) {
+            if (shapeMember.isStreaming() || childShape.isBlob() || childShape.isString()) {
+                return "Aws::Utils::Stream::ResponseStream";
+            }
+        }
+        return computeCborCppType(childShape);
+    }
+
     public static String computeJsonCppType(Shape shape) {
         if(shape.isTimeStamp() && shape.getTimestampFormat() != null) {
             return CORAL_TYPE_TO_JSON_CPP_TYPE_MAPPING.get(shape.getTimestampFormat().toLowerCase());
@@ -387,6 +432,9 @@ public static Set<String> computeHeaderIncludes(String projectName, Shape shape)
                     headers.add(formatModelIncludeName(projectName, shapeInList));
                 }
             }
+            if (next.isSparse()) {
+                headers.add("<aws/crt/Optional.h>");
+            }
             if(!next.isPrimitive()) {
                 if (next.isException() && !next.isModeledException()) {
                     // C++ SDK code generator skips generating exceptions that can be expressed using

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/generators/cpp/CppProtocolTestGenerator.java

@@ -11,7 +11,6 @@
 
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.Shape;
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.ShapeMember;
-import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.cpp.CppCborViewHelper;
 import com.amazonaws.util.awsclientgenerator.domainmodels.codegeneration.cpp.CppViewHelper;
 import com.amazonaws.util.awsclientgenerator.domainmodels.protocol_test.ProtocolTestModel;
 import com.amazonaws.util.awsclientgenerator.domainmodels.protocol_test.ProtocolTestSuite;
@@ -115,11 +114,7 @@ protected final VelocityContext createContext() {
         context.put("testModel", testModel);
         context.put("input.encoding", StandardCharsets.UTF_8.name());
         context.put("output.encoding", StandardCharsets.UTF_8.name());
-        if(serviceModel.getMetadata().getProtocol().equals("smithy-rpc-v2-cbor")){
-            context.put("CppViewHelper", CppCborViewHelper.class);
-        } else {
-            context.put("CppViewHelper", CppViewHelper.class);
-        }
+        context.put("CppViewHelper", CppViewHelper.class);
         return context;
     }
 

tools/code-generation/generator/src/main/java/com/amazonaws/util/awsclientgenerator/transform/C2jModelToGeneratorModelTransformer.java

@@ -439,6 +439,10 @@ Shape convertShapeBasics(C2jShape c2jShape, String shapeName) {
         shape.setEvent(c2jShape.isEvent());
         shape.setException(c2jShape.isException());
         shape.setDocument(c2jShape.isDocument());
+        shape.setSparse(c2jShape.isSparse());
+        if (shape.isSparse() && !shape.isList() && !shape.isMap()) {
+            throw new SourceGenerationFailedException("The sparse trait is only applicable to list and map shapes, but was found on shape: " + shape.getName());
+        }
 
         if (c2jShape.getXmlNamespace() != null) {
             XmlNamespace xmlns = new XmlNamespace();
@@ -850,6 +854,7 @@ Shape cloneShape(Shape shape) {
         cloned.setException(shape.isException());
         cloned.setXmlNamespace(shape.getXmlNamespace());
         cloned.setDocument(shape.isDocument());
+        cloned.setSparse(shape.isSparse());
         return cloned;
     }
 

tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/ModelClassHeaderMembersSource.vm

@@ -42,7 +42,11 @@
   ${spaces}for(const auto& item : $memberVarName)
   ${spaces}{
   ${spaces}  ss << "${locationName}" << item.first;
+#if($member.shape.sparse)
+  ${spaces}  if(item.second.has_value()) { headers.emplace(ss.str(), item.second.value()); }
+#else
   ${spaces}  headers.emplace(ss.str(), item.second);
+#end
   ${spaces}  ss.str("");
   ${spaces}}
 #elseif($member.shape.list)

tools/code-generation/generator/src/main/resources/com/amazonaws/util/awsclientgenerator/velocity/cpp/ModelClassMembersAndInlines.vm

@@ -111,6 +111,24 @@
 #set($mapMember = $member.value.shape)
 #set($keyType = $CppViewHelper.computeCppType($mapMember.mapKey.shape))
 #set($valueType = $CppViewHelper.computeCppType($mapMember.mapValue.shape))
+#if($mapMember.sparse)
+#set($optionalValueType = "Aws::Crt::Optional<${valueType}>")
+#if(!$mapMember.mapKey.shape.primitive && !$mapMember.mapKey.shape.enum && !$mapMember.mapValue.shape.primitive && !$mapMember.mapValue.shape.enum)
+#set($keyTemplType = "${memberKeyWithFirstLetterCapitalized}KeyT")
+#set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}ValueT")
+    template<typename ${keyTemplType} = $keyType, typename ${valueTemplType} = $optionalValueType>
+    ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyTemplType}&& key, ${valueTemplType}&& value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(std::forward<${keyTemplType}>(key), std::forward<${valueTemplType}>(value)); return *this;
+    }
+#else
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyType} key, ${valueType} value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(key, value); return *this;
+    }
+#end
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${keyType} key, ${optionalValueType} value) {
+      ${setHasBeenSet}${memberVariableName}.emplace(key, value); return *this;
+    }
+#else
 #if(!$mapMember.mapKey.shape.primitive && !$mapMember.mapKey.shape.enum && !$mapMember.mapValue.shape.primitive && !$mapMember.mapValue.shape.enum)
 #set($keyTemplType = "${memberKeyWithFirstLetterCapitalized}KeyT")
 #set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}ValueT")
@@ -124,16 +142,29 @@
     }
 #end
 #end
+#end
 #if($member.value.shape.list)
 #set($listMember = $member.value.shape)
 #set($valueType = $CppViewHelper.computeCppType($listMember.listMember.shape))
+#if($listMember.sparse)
+#set($optionalValueType = "Aws::Crt::Optional<${valueType}>")
+#if(!$listMember.listMember.shape.primitive && !$listMember.listMember.shape.enum)
+#set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}T")
+    template<typename ${valueTemplType} = $valueType>
+    ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueTemplType}&& value) { ${setHasBeenSet}${memberVariableName}.emplace_back(std::forward<${valueTemplType}>(value)); return *this; }
+#else
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueType} value)$adderWithCopy
+#end
+    ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${optionalValueType} value)$adderWithCopy
+#else
 #if(!$listMember.listMember.shape.primitive && !$listMember.listMember.shape.enum)
 #set($valueTemplType = "${memberKeyWithFirstLetterCapitalized}T")
     template<typename ${valueTemplType} = $valueType>
     ${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueTemplType}&& value) { ${setHasBeenSet}${memberVariableName}.emplace_back(std::forward<${valueTemplType}>(value)); return *this; }
 #else
     ${inline}${classNameRef} Add${memberKeyWithFirstLetterCapitalized}(${valueType} value)$adderWithCopy
 #end
+#end
 #end
     ///@}
 #end##if(!($CppViewHelper.isStreamingPayloadMember($shape, $member.key) && $shape.request))