Skip to content

Commit 74a9d11

Browse files
slissnermhalbritter
slissner
authored and
mhalbritter
committed
Add Graylog Extended Log Format (GELF) for structured logging
See gh-42158
1 parent 915fc2e commit 74a9d11

File tree

12 files changed

+747
-2
lines changed

12 files changed

+747
-2
lines changed

‎spring-boot-project/spring-boot-docs/src/docs/antora/modules/reference/pages/features/logging.adoc

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ Structured logging is a technique where the log output is written in a well-defi
445445
Spring Boot supports structured logging and has support for the following JSON formats out of the box:
446446

447447
* xref:#features.logging.structured.ecs[Elastic Common Schema (ECS)]
448+
* xref:#features.logging.structured.gelf[Graylog Extended Log Format (GELF)]
448449
* xref:#features.logging.structured.logstash[Logstash]
449450

450451
To enable structured logging, set the property configprop:logging.structured.format.console[] (for console output) or configprop:logging.structured.format.file[] (for file output) to the id of the format you want to use.
@@ -492,6 +493,53 @@ logging:
492493

493494
NOTE: configprop:logging.structured.ecs.service.name[] will default to configprop:spring.application.name[] if not specified.
494495

496+
NOTE: configprop:logging.structured.ecs.service.version[] will default to configprop:spring.application.version[] if not specified.
497+
498+
499+
500+
[[features.logging.structured.gelf]]
501+
=== Graylog Extended Log Format (GELF)
502+
https://go2docs.graylog.org/current/getting_in_log_data/gelf.html[Graylog Extended Log Format] is a JSON based logging format for the Graylog log analytics platform.
503+
504+
To enable the Graylog Extended Log Format, set the appropriate `format` property to `gelf`:
505+
506+
[configprops,yaml]
507+
----
508+
logging:
509+
structured:
510+
format:
511+
console: gelf
512+
file: gelf
513+
----
514+
515+
A log line looks like this:
516+
517+
[source,json]
518+
----
519+
{"version":"1.1","short_message":"Hello structured logging!","timestamp":1.725530750186E9,"level":6,"_level_name":"INFO","_process_pid":9086,"_process_thread_name":"main","host":"spring-boot-gelf","_log_logger":"com.slissner.springbootgelf.ExampleLogger","_userId":"1","_testkey_testmessage":"test"}
520+
----
521+
522+
This format also adds every key value pair contained in the MDC to the JSON object.
523+
You can also use the https://www.slf4j.org/manual.html#fluent[SLF4J fluent logging API] to add key value pairs to the logged JSON object with the https://www.slf4j.org/apidocs/org/slf4j/spi/LoggingEventBuilder.html#addKeyValue(java.lang.String,java.lang.Object)[addKeyValue] method.
524+
525+
The `service` values can be customized using `logging.structured.gelf.service` properties:
526+
527+
[configprops,yaml]
528+
----
529+
logging:
530+
structured:
531+
gelf:
532+
service:
533+
name: MyService
534+
version: 1.0
535+
environment: Production
536+
node-name: Primary
537+
----
538+
539+
NOTE: configprop:logging.structured.gelf.service.name[] will default to configprop:spring.application.name[] if not specified.
540+
541+
NOTE: configprop:logging.structured.gelf.service.version[] will default to configprop:spring.application.version[] if not specified.
542+
495543

496544

497545
[[features.logging.structured.logstash]]

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/GraylogExtendedLogFormatStructuredLogFormatter.java

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
* Copyright 2012-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.boot.logging.log4j2;
18+
19+
import java.math.BigDecimal;
20+
import java.util.Objects;
21+
import java.util.Set;
22+
import java.util.function.Function;
23+
import java.util.regex.Pattern;
24+
25+
import org.apache.logging.log4j.Level;
26+
import org.apache.logging.log4j.core.LogEvent;
27+
import org.apache.logging.log4j.core.impl.ThrowableProxy;
28+
import org.apache.logging.log4j.core.net.Severity;
29+
import org.apache.logging.log4j.core.time.Instant;
30+
import org.apache.logging.log4j.message.Message;
31+
import org.apache.logging.log4j.util.ReadOnlyStringMap;
32+
33+
import org.springframework.boot.json.JsonWriter;
34+
import org.springframework.boot.logging.structured.CommonStructuredLogFormat;
35+
import org.springframework.boot.logging.structured.GraylogExtendedLogFormatService;
36+
import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter;
37+
import org.springframework.boot.logging.structured.StructuredLogFormatter;
38+
import org.springframework.core.env.Environment;
39+
import org.springframework.util.Assert;
40+
import org.springframework.util.ObjectUtils;
41+
42+
/**
43+
* Log4j2 {@link StructuredLogFormatter} for
44+
* {@link CommonStructuredLogFormat#GRAYLOG_EXTENDED_LOG_FORMAT}. Supports GELF version
45+
* 1.1.
46+
*
47+
* @author Samuel Lissner
48+
*/
49+
class GraylogExtendedLogFormatStructuredLogFormatter extends JsonWriterStructuredLogFormatter<LogEvent> {
50+
51+
/**
52+
* Allowed characters in field names are any word character (letter, number,
53+
* underscore), dashes and dots.
54+
*/
55+
private static final Pattern FIELD_NAME_VALID_PATTERN = Pattern.compile("^[\\w\\.\\-]*$");
56+
57+
/**
58+
* Every field been sent and prefixed with an underscore "_" will be treated as an
59+
* additional field.
60+
*/
61+
private static final String ADDITIONAL_FIELD_PREFIX = "_";
62+
63+
/**
64+
* Libraries SHOULD not allow to send id as additional field ("_id"). Graylog server
65+
* nodes omit this field automatically.
66+
*/
67+
private static final Set<String> ADDITIONAL_FIELD_ILLEGAL_KEYS = Set.of("_id");
68+
69+
/**
70+
* Default format to be used for the `full_message` property when there is a throwable
71+
* present in the log event.
72+
*/
73+
private static final String DEFAULT_FULL_MESSAGE_WITH_THROWABLE_FORMAT = "%s%n%n%s";
74+
75+
GraylogExtendedLogFormatStructuredLogFormatter(Environment environment) {
76+
super((members) -> jsonMembers(environment, members));
77+
}
78+
79+
private static void jsonMembers(Environment environment, JsonWriter.Members<LogEvent> members) {
80+
members.add("version", "1.1");
81+
82+
// note: a blank message will lead to a Graylog error as of Graylog v6.0.x. We are
83+
// ignoring this here.
84+
members.add("short_message", LogEvent::getMessage).as(Message::getFormattedMessage);
85+
86+
members.add("timestamp", LogEvent::getInstant)
87+
.as(GraylogExtendedLogFormatStructuredLogFormatter::formatTimeStamp);
88+
members.add("level", GraylogExtendedLogFormatStructuredLogFormatter::convertLevel);
89+
members.add("_level_name", LogEvent::getLevel).as(Level::name);
90+
91+
members.add("_process_pid", environment.getProperty("spring.application.pid", Long.class))
92+
.when(Objects::nonNull);
93+
members.add("_process_thread_name", LogEvent::getThreadName);
94+
95+
GraylogExtendedLogFormatService.get(environment).jsonMembers(members);
96+
97+
members.add("_log_logger", LogEvent::getLoggerName);
98+
99+
members.from(LogEvent::getContextData)
100+
.whenNot(ReadOnlyStringMap::isEmpty)
101+
.usingPairs((contextData, pairs) -> contextData
102+
.forEach((key, value) -> pairs.accept(makeAdditionalFieldName(key), value)));
103+
104+
members.add().whenNotNull(LogEvent::getThrownProxy).usingMembers((eventMembers) -> {
105+
final Function<LogEvent, ThrowableProxy> throwableProxyGetter = LogEvent::getThrownProxy;
106+
107+
eventMembers.add("full_message",
108+
GraylogExtendedLogFormatStructuredLogFormatter::formatFullMessageWithThrowable);
109+
eventMembers.add("_error_type", throwableProxyGetter.andThen(ThrowableProxy::getThrowable))
110+
.whenNotNull()
111+
.as(ObjectUtils::nullSafeClassName);
112+
eventMembers.add("_error_stack_trace",
113+
throwableProxyGetter.andThen(ThrowableProxy::getExtendedStackTraceAsString));
114+
eventMembers.add("_error_message", throwableProxyGetter.andThen(ThrowableProxy::getMessage));
115+
});
116+
}
117+
118+
/**
119+
* GELF requires "seconds since UNIX epoch with optional <b>decimal places for
120+
* milliseconds</b>". To comply with this requirement, we format a POSIX timestamp
121+
* with millisecond precision as e.g. "1725459730385" -> "1725459730.385"
122+
* @param timeStamp the timestamp of the log message. Note it is not the standard Java
123+
* `Instant` type but {@link org.apache.logging.log4j.core.time}
124+
* @return the timestamp formatted as string with millisecond precision
125+
*/
126+
private static double formatTimeStamp(final Instant timeStamp) {
127+
return new BigDecimal(timeStamp.getEpochMillisecond()).movePointLeft(3).doubleValue();
128+
}
129+
130+
/**
131+
* Converts the log4j2 event level to the Syslog event level code.
132+
* @param event the log event
133+
* @return an integer representing the syslog log level code
134+
* @see Severity class from Log4j2 which contains the conversion logic
135+
*/
136+
private static int convertLevel(final LogEvent event) {
137+
return Severity.getSeverity(event.getLevel()).getCode();
138+
}
139+
140+
private static String formatFullMessageWithThrowable(final LogEvent event) {
141+
return String.format(DEFAULT_FULL_MESSAGE_WITH_THROWABLE_FORMAT, event.getMessage().getFormattedMessage(),
142+
event.getThrownProxy().getExtendedStackTraceAsString());
143+
}
144+
145+
private static String makeAdditionalFieldName(String fieldName) {
146+
Assert.notNull(fieldName, "fieldName must not be null");
147+
Assert.isTrue(FIELD_NAME_VALID_PATTERN.matcher(fieldName).matches(),
148+
() -> String.format("fieldName must be a valid according to GELF standard. [fieldName=%s]", fieldName));
149+
Assert.isTrue(!ADDITIONAL_FIELD_ILLEGAL_KEYS.contains(fieldName), () -> String.format(
150+
"fieldName must not be an illegal additional field key according to GELF standard. [fieldName=%s]",
151+
fieldName));
152+
153+
if (fieldName.startsWith(ADDITIONAL_FIELD_PREFIX)) {
154+
// No need to prepend the `ADDITIONAL_FIELD_PREFIX` in case the caller already
155+
// has prepended the prefix.
156+
return fieldName;
157+
}
158+
159+
return ADDITIONAL_FIELD_PREFIX + fieldName;
160+
}
161+
162+
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/log4j2/StructuredLogLayout.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ private void addCommonFormatters(CommonFormatters<LogEvent> commonFormatters) {
105105
commonFormatters.add(CommonStructuredLogFormat.ELASTIC_COMMON_SCHEMA,
106106
(instantiator) -> new ElasticCommonSchemaStructuredLogFormatter(
107107
instantiator.getArg(Environment.class)));
108+
commonFormatters.add(CommonStructuredLogFormat.GRAYLOG_EXTENDED_LOG_FORMAT,
109+
(instantiator) -> new GraylogExtendedLogFormatStructuredLogFormatter(
110+
instantiator.getArg(Environment.class)));
108111
commonFormatters.add(CommonStructuredLogFormat.LOGSTASH,
109112
(instantiator) -> new LogstashStructuredLogFormatter());
110113
}

‎spring-boot-project/spring-boot/src/main/java/org/springframework/boot/logging/logback/GraylogExtendedLogFormatStructuredLogFormatter.java

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright 2012-2024 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.boot.logging.logback;
18+
19+
import java.math.BigDecimal;
20+
import java.util.Map;
21+
import java.util.Objects;
22+
import java.util.Set;
23+
import java.util.function.Function;
24+
import java.util.regex.Pattern;
25+
import java.util.stream.Collectors;
26+
27+
import ch.qos.logback.classic.pattern.ThrowableProxyConverter;
28+
import ch.qos.logback.classic.spi.ILoggingEvent;
29+
import ch.qos.logback.classic.spi.IThrowableProxy;
30+
import ch.qos.logback.classic.util.LevelToSyslogSeverity;
31+
import org.slf4j.event.KeyValuePair;
32+
33+
import org.springframework.boot.json.JsonWriter;
34+
import org.springframework.boot.json.JsonWriter.PairExtractor;
35+
import org.springframework.boot.logging.structured.CommonStructuredLogFormat;
36+
import org.springframework.boot.logging.structured.GraylogExtendedLogFormatService;
37+
import org.springframework.boot.logging.structured.JsonWriterStructuredLogFormatter;
38+
import org.springframework.boot.logging.structured.StructuredLogFormatter;
39+
import org.springframework.core.env.Environment;
40+
import org.springframework.util.Assert;
41+
42+
/**
43+
* Logback {@link StructuredLogFormatter} for
44+
* {@link CommonStructuredLogFormat#GRAYLOG_EXTENDED_LOG_FORMAT}. Supports GELF version
45+
* 1.1.
46+
*
47+
* @author Samuel Lissner
48+
*/
49+
class GraylogExtendedLogFormatStructuredLogFormatter extends JsonWriterStructuredLogFormatter<ILoggingEvent> {
50+
51+
/**
52+
* Allowed characters in field names are any word character (letter, number,
53+
* underscore), dashes and dots.
54+
*/
55+
private static final Pattern FIELD_NAME_VALID_PATTERN = Pattern.compile("^[\\w\\.\\-]*$");
56+
57+
/**
58+
* Every field been sent and prefixed with an underscore "_" will be treated as an
59+
* additional field.
60+
*/
61+
private static final String ADDITIONAL_FIELD_PREFIX = "_";
62+
63+
/**
64+
* Libraries SHOULD not allow to send id as additional field ("_id"). Graylog server
65+
* nodes omit this field automatically.
66+
*/
67+
private static final Set<String> ADDITIONAL_FIELD_ILLEGAL_KEYS = Set.of("_id");
68+
69+
/**
70+
* Default format to be used for the `full_message` property when there is a throwable
71+
* present in the log event.
72+
*/
73+
private static final String DEFAULT_FULL_MESSAGE_WITH_THROWABLE_FORMAT = "%s%n%n%s";
74+
75+
private static final PairExtractor<KeyValuePair> keyValuePairExtractor = PairExtractor
76+
.of((pair) -> makeAdditionalFieldName(pair.key), (pair) -> pair.value);
77+
78+
GraylogExtendedLogFormatStructuredLogFormatter(Environment environment,
79+
ThrowableProxyConverter throwableProxyConverter) {
80+
super((members) -> jsonMembers(environment, throwableProxyConverter, members));
81+
}
82+
83+
private static void jsonMembers(Environment environment, ThrowableProxyConverter throwableProxyConverter,
84+
JsonWriter.Members<ILoggingEvent> members) {
85+
members.add("version", "1.1");
86+
87+
// note: a blank message will lead to a Graylog error as of Graylog v6.0.x. We are
88+
// ignoring this here.
89+
members.add("short_message", ILoggingEvent::getFormattedMessage);
90+
91+
members.add("timestamp", ILoggingEvent::getTimeStamp)
92+
.as(GraylogExtendedLogFormatStructuredLogFormatter::formatTimeStamp);
93+
members.add("level", LevelToSyslogSeverity::convert);
94+
members.add("_level_name", ILoggingEvent::getLevel);
95+
96+
members.add("_process_pid", environment.getProperty("spring.application.pid", Long.class))
97+
.when(Objects::nonNull);
98+
members.add("_process_thread_name", ILoggingEvent::getThreadName);
99+
100+
GraylogExtendedLogFormatService.get(environment).jsonMembers(members);
101+
102+
members.add("_log_logger", ILoggingEvent::getLoggerName);
103+
104+
members.addMapEntries(mapMDCProperties(ILoggingEvent::getMDCPropertyMap));
105+
106+
members.from(ILoggingEvent::getKeyValuePairs)
107+
.whenNotEmpty()
108+
.usingExtractedPairs(Iterable::forEach, keyValuePairExtractor);
109+
110+
members.add().whenNotNull(ILoggingEvent::getThrowableProxy).usingMembers((throwableMembers) -> {
111+
throwableMembers.add("full_message",
112+
(event) -> formatFullMessageWithThrowable(throwableProxyConverter, event));
113+
throwableMembers.add("_error_type", ILoggingEvent::getThrowableProxy).as(IThrowableProxy::getClassName);
114+
throwableMembers.add("_error_stack_trace", throwableProxyConverter::convert);
115+
throwableMembers.add("_error_message", ILoggingEvent::getThrowableProxy).as(IThrowableProxy::getMessage);
116+
});
117+
}
118+
119+
/**
120+
* GELF requires "seconds since UNIX epoch with optional <b>decimal places for
121+
* milliseconds</b>". To comply with this requirement, we format a POSIX timestamp
122+
* with millisecond precision as e.g. "1725459730385" -> "1725459730.385"
123+
* @param timeStamp the timestamp of the log message
124+
* @return the timestamp formatted as string with millisecond precision
125+
*/
126+
private static double formatTimeStamp(final long timeStamp) {
127+
return new BigDecimal(timeStamp).movePointLeft(3).doubleValue();
128+
}
129+
130+
private static String formatFullMessageWithThrowable(final ThrowableProxyConverter throwableProxyConverter,
131+
ILoggingEvent event) {
132+
return String.format(DEFAULT_FULL_MESSAGE_WITH_THROWABLE_FORMAT, event.getFormattedMessage(),
133+
throwableProxyConverter.convert(event));
134+
}
135+
136+
private static Function<ILoggingEvent, Map<String, String>> mapMDCProperties(
137+
Function<ILoggingEvent, Map<String, String>> MDCPropertyMapGetter) {
138+
return MDCPropertyMapGetter.andThen((mdc) -> mdc.entrySet()
139+
.stream()
140+
.collect(Collectors.toMap((entry) -> makeAdditionalFieldName(entry.getKey()), Map.Entry::getValue)));
141+
}
142+
143+
private static String makeAdditionalFieldName(String fieldName) {
144+
Assert.notNull(fieldName, "fieldName must not be null");
145+
Assert.isTrue(FIELD_NAME_VALID_PATTERN.matcher(fieldName).matches(),
146+
() -> String.format("fieldName must be a valid according to GELF standard. [fieldName=%s]", fieldName));
147+
Assert.isTrue(!ADDITIONAL_FIELD_ILLEGAL_KEYS.contains(fieldName), () -> String.format(
148+
"fieldName must not be an illegal additional field key according to GELF standard. [fieldName=%s]",
149+
fieldName));
150+
151+
if (fieldName.startsWith(ADDITIONAL_FIELD_PREFIX)) {
152+
// No need to prepend the `ADDITIONAL_FIELD_PREFIX` in case the caller already
153+
// has prepended the prefix.
154+
return fieldName;
155+
}
156+
157+
return ADDITIONAL_FIELD_PREFIX + fieldName;
158+
}
159+
160+
}

0 commit comments

Comments
 (0)