Change Custom Fields JSON key to #cf

This is the first step towards the new custom fields approach. Since
#cf is not a valid Java variable name, the code generation needed to be
changed. Based on the new fields.yml code and documentation were
generated. The failing unit test were adapted to the new JSON key.

Author: KarstenSchnitter

cf-java-logging-support-core/beats/app-logs/docs/fields.asciidoc

@@ -336,9 +336,23 @@ required: False
 A list of names to further categorize this log message.
 
 
-=== custom_fields Fields
+==== #cf
 
-A collection of non-standard fields as key-value pairs.
+type: object
+
+example: "#cf": {
+  "string": [
+    {"l":"some_label", "v":"some_value", "i": 0},
+    {"l":"other_label", "v":"other_value", "i": 1}
+  ]
+}
+
+
+required: False
+
+An object containing collections of non-standard fields.
+The field "string" contains custom fields with label "l", value "v" and an index "i".
+The index can be used for field order during parsing.
 
 NOTE: As this is "custom" there are no predefined fields here!
 

cf-java-logging-support-core/beats/app-logs/etc/fields.yml

@@ -249,11 +249,20 @@ app-logs:
       description: |
         A list of names to further categorize this log message.
 
-    - name: "custom_fields"
-      type: group
+    - name: "#cf"
+      java_alias: "CUSTOM_FIELDS"
+      type: object
       required: false
-      example: "cutom_fields: {\"some_key\": \"some_value\"}"
+      example: |
+        "#cf": {
+          "string": [
+            {"l":"some_label", "v":"some_value", "i": 0},
+            {"l":"other_label", "v":"other_value", "i": 1}
+          ]
+        }
       description: |
-        A collection of non-standard fields as key-value pairs.
+        An object containing collections of non-standard fields.
+        The field "string" contains custom fields with label "l", value "v" and an index "i".
+        The index can be used for field order during parsing.
 
         NOTE: As this is "custom" there are no predefined fields here!

cf-java-logging-support-core/beats/scripts/gen_java_fields.rb

@@ -17,7 +17,9 @@
   spec.keys.each do |k|
     if spec[k].class == Hash && spec[k].has_key?('fields')
       spec[k]['fields'].each do |f|
-        fields.add(f['name'])
+        if fields.select { |c| c['name'].upcase == f['name'].upcase}.empty?
+          fields.add(f)
+        end
       end
     end
   end
@@ -37,7 +39,8 @@
 EOD
 
 fields.each do |f|
-  puts "  public String #{f.upcase} = \"#{f}\";"
+  key = f.has_key?('java_alias') ? f['java_alias'] : f['name']
+  puts "  public String #{key.upcase} = \"#{f['name']}\";"
 end
 
 puts "}"

cf-java-logging-support-core/src/main/java/com/sap/hcp/cf/logging/common/Fields.java

@@ -32,7 +32,7 @@ public interface Fields {
   public String MSG = "msg";
   public String STACKTRACE = "stacktrace";
   public String CATEGORIES = "categories";
-  public String CUSTOM_FIELDS = "custom_fields";
+  public String CUSTOM_FIELDS = "#cf";
   public String REQUEST = "request";
   public String REQUEST_SENT_AT = "request_sent_at";
   public String REQUEST_RECEIVED_AT = "request_received_at";

cf-java-logging-support-log4j2/src/test/java/com/sap/hcp/cf/log4j2/layout/LayoutPatternBuilderTest.java

@@ -51,7 +51,7 @@ public void contextProperties() throws Exception {
 	public void customFields() throws Exception {
 		String pattern = new LayoutPatternBuilder().addCustomFields(Arrays.asList("this key", "that key")).build();
 
-		assertThat(pattern, specificPart(is(",\"custom_fields\":{%cf{this key}{that key}}")));
+		assertThat(pattern, specificPart(is(",\"#cf\":{%cf{this key}{that key}}")));
 	}
 
 	@Test
@@ -83,7 +83,7 @@ public void applicationScenario() throws Exception {
 				.suppressExceptions().build();
 
 		assertThat(pattern, specificPart(is(
-				",\"type\":\"log\",\"logger\":\"%replace{%logger}{\"}{\\\\\"}\",\"thread\":\"%replace{%thread}{\"}{\\\\\"}\",\"level\":\"%p\",\"categories\":%categories,\"msg\":%jsonmsg{escape},%ctxp{excluded-field},\"custom_fields\":{%cf{custom-field}}%ex{0} ")));
+				",\"type\":\"log\",\"logger\":\"%replace{%logger}{\"}{\\\\\"}\",\"thread\":\"%replace{%thread}{\"}{\\\\\"}\",\"level\":\"%p\",\"categories\":%categories,\"msg\":%jsonmsg{escape},%ctxp{excluded-field},\"#cf\":{%cf{custom-field}}%ex{0} ")));
 	}
 
 	@Test
@@ -93,7 +93,7 @@ public void exceptionScenario() throws Exception {
 				.build();
 
 		assertThat(pattern, specificPart(is(
-				",\"type\":\"log\",\"logger\":\"%replace{%logger}{\"}{\\\\\"}\",\"thread\":\"%replace{%thread}{\"}{\\\\\"}\",\"level\":\"%p\",\"categories\":%categories,\"msg\":%jsonmsg{escape},%ctxp{excluded-field},\"custom_fields\":{%cf{custom-field}},\"stacktrace\":%stacktrace")));
+				",\"type\":\"log\",\"logger\":\"%replace{%logger}{\"}{\\\\\"}\",\"thread\":\"%replace{%thread}{\"}{\\\\\"}\",\"level\":\"%p\",\"categories\":%categories,\"msg\":%jsonmsg{escape},%ctxp{excluded-field},\"#cf\":{%cf{custom-field}},\"stacktrace\":%stacktrace")));
 	}
 
 	private static Matcher<String> specificPart(Matcher<String> expected) {

cf-java-logging-support-log4j2/src/test/java/com/sap/hcp/cf/logging/common/AbstractTest.java

@@ -66,7 +66,7 @@ private String getLastLine() {
     }
 
     protected String getCustomField(String fieldName) {
-        Map<String, Object> cfMap = getMap("custom_fields");
+		Map<String, Object> cfMap = getMap(Fields.CUSTOM_FIELDS);
         Object fObj = cfMap.get(fieldName);
         if (fObj != null) {
             return fObj.toString();

cf-java-logging-support-log4j2/src/test/java/com/sap/hcp/cf/logging/common/TestCustomFields.java

@@ -86,7 +86,7 @@ public void testOrderOfLogMessageWithTwoCustomFields() {
     }
 
     private String getCustomFields() {
-        return getField("custom_fields");
+		return getField(Fields.CUSTOM_FIELDS);
     }
 
 	@Test