Support MultiformatMessage JSON format (#30)

Author: felixbarny

log4j2-ecs-layout/README.md

@@ -46,3 +46,46 @@ set the `includeMarkers` attribute to `true` (default: `false`).
     </Loggers>
 </Configuration>
 ```
+
+## Structured logging
+
+By leveraging log4j2's `MapMessage` or even by implementing your own `MultiformatMessage` with JSON support,
+you can add additional fields to the resulting JSON.
+
+Example:
+
+```java
+logger.info(new StringMapMessage().with("message", "foo").with("foo", "bar"));
+``` 
+
+### Gotchas
+
+A common pitfall is how dots in field names are handled in Elasticsearch and how they affect the mapping.
+In recent Elasticsearch versions, the following JSON structures would result in the same index mapping:
+
+```json
+{
+  "foo.bar": "baz"
+}
+```
+
+```json
+{
+  "foo": {
+    "bar": "baz"
+  }
+}
+```
+The property `foo` would be mapped to the [Object datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html).
+
+This means that you can't index a document where `foo` would be a different datatype, as in shown in the following example:
+
+```json
+{
+  "foo": "bar"
+}
+```
+
+In that example, `foo` is a string.
+Trying to index that document results in an error because the data type of `foo` can't be object and string at the same time.
+ 

log4j2-ecs-layout/src/main/java/co/elastic/logging/log4j2/EcsLayout.java

@@ -42,8 +42,9 @@
 import org.apache.logging.log4j.core.lookup.StrSubstitutor;
 import org.apache.logging.log4j.core.util.KeyValuePair;
 import org.apache.logging.log4j.core.util.StringBuilderWriter;
-import org.apache.logging.log4j.message.MapMessage;
 import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.message.MultiformatMessage;
+import org.apache.logging.log4j.util.MultiFormatStringBuilderFormattable;
 import org.apache.logging.log4j.util.StringBuilderFormattable;
 import org.apache.logging.log4j.util.TriConsumer;
 
@@ -55,12 +56,15 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 @Plugin(name = "EcsLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE)
 public class EcsLayout extends AbstractStringLayout {
 
     private static final ThreadLocal<StringBuilder> messageStringBuilder = new ThreadLocal<StringBuilder>();
     public static final Charset UTF_8 = Charset.forName("UTF-8");
+    public static final String[] JSON_FORMAT = {"JSON"};
 
     private final TriConsumer<String, Object, StringBuilder> WRITE_KEY_VALUES_INTO = new TriConsumer<String, Object, StringBuilder>() {
         @Override
@@ -80,6 +84,7 @@ public void accept(final String key, final Object value, final StringBuilder str
     private final Set<String> topLevelLabels;
     private String serviceName;
     private boolean includeMarkers;
+    private final ConcurrentMap<Class<? extends MultiformatMessage>, Boolean> supportsJson = new ConcurrentHashMap<Class<? extends MultiformatMessage>, Boolean>();
 
     private EcsLayout(Configuration config, String serviceName, boolean includeMarkers, KeyValuePair[] additionalFields, Collection<String> topLevelLabels) {
         super(config, UTF_8, null, null);
@@ -201,6 +206,37 @@ private void serializeMarker(StringBuilder builder, Marker marker) {
     }
 
     private void serializeMessage(StringBuilder builder, boolean gcFree, Message message, Throwable thrown) {
+        if (message instanceof MultiformatMessage) {
+            MultiformatMessage multiformatMessage = (MultiformatMessage) message;
+            if (supportsJson(multiformatMessage)) {
+                serializeJsonMessage(builder, multiformatMessage);
+            } else {
+                serializeSimpleMessage(builder, gcFree, message, thrown);
+            }
+        } else {
+            serializeSimpleMessage(builder, gcFree, message, thrown);
+        }
+    }
+
+    private void serializeJsonMessage(StringBuilder builder, MultiformatMessage message) {
+        final StringBuilder messageBuffer = getMessageStringBuilder();
+        if (message instanceof MultiFormatStringBuilderFormattable) {
+            ((MultiFormatStringBuilderFormattable) message).formatTo(JSON_FORMAT, messageBuffer);
+        } else {
+            messageBuffer.append(message.getFormattedMessage(JSON_FORMAT));
+        }
+        if (isObject(messageBuffer)) {
+            moveToRoot(messageBuffer);
+            builder.append(messageBuffer);
+            builder.append(", ");
+        } else {
+            builder.append("\"message\":");
+            builder.append(messageBuffer);
+            builder.append(", ");
+        }
+    }
+
+    private void serializeSimpleMessage(StringBuilder builder, boolean gcFree, Message message, Throwable thrown) {
         builder.append("\"message\":\"");
         if (message instanceof CharSequence) {
             JsonUtils.quoteAsString(((CharSequence) message), builder);
@@ -220,10 +256,30 @@ private void serializeMessage(StringBuilder builder, boolean gcFree, Message mes
             JsonUtils.quoteAsString(formatThrowable(thrown), builder);
         }
         builder.append("\", ");
-        if (message instanceof MapMessage) {
-            MapMessage mapMessage = (MapMessage) message;
-            mapMessage.forEach(WRITE_KEY_VALUES_INTO, builder);
+    }
+
+    private boolean isObject(StringBuilder messageBuffer) {
+        return messageBuffer.length() > 1 && messageBuffer.charAt(0) == '{' && messageBuffer.charAt(messageBuffer.length() -1) == '}';
+    }
+
+    private void moveToRoot(StringBuilder messageBuffer) {
+        messageBuffer.setCharAt(0, ' ');
+        messageBuffer.setCharAt(messageBuffer.length() -1, ' ');
+    }
+
+    private boolean supportsJson(MultiformatMessage message) {
+        Boolean supportsJson = this.supportsJson.get(message.getClass());
+        if (supportsJson == null) {
+            supportsJson = false;
+            for (String format : message.getFormats()) {
+                if (format.equalsIgnoreCase("JSON")) {
+                    supportsJson = true;
+                    break;
+                }
+            }
+            this.supportsJson.put(message.getClass(), supportsJson);
         }
+        return supportsJson;
     }
 
     private static CharSequence formatThrowable(final Throwable throwable) {

log4j2-ecs-layout/src/test/java/co/elastic/logging/log4j2/AbstractLog4j2EcsLayoutTest.java

@@ -25,6 +25,7 @@
 package co.elastic.logging.log4j2;
 
 import co.elastic.logging.AbstractEcsLoggingTest;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import org.apache.logging.log4j.Marker;
 import org.apache.logging.log4j.MarkerManager;
@@ -35,8 +36,6 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 
-import java.util.Map;
-
 import static org.assertj.core.api.Assertions.assertThat;
 
 abstract class AbstractLog4j2EcsLayoutTest extends AbstractEcsLoggingTest {
@@ -72,8 +71,10 @@ void testMarker() throws Exception {
 
     @Test
     void testMapMessage() throws Exception {
-        root.info(new StringMapMessage(Map.of("foo", "bar")));
-        assertThat(getLastLogLine().get("labels.foo").textValue()).isEqualTo("bar");
+        root.info(new StringMapMessage().with("message", "foo").with("foo", "bar"));
+        JsonNode log = getLastLogLine();
+        assertThat(log.get("message").textValue()).isEqualTo("foo");
+        assertThat(log.get("foo").textValue()).isEqualTo("bar");
     }
 
     @Override