opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add new Full Message JSON field to the Cloud Trail input (#24786)

Browse Source 
* Add new Full Message JSON field to the Cloud Trail input

The existing full_message field only contains a stringified Java array, which is not parsable. With the new `full_message_json` field, it should be possible to use pipeline functions to parse the message content and extract specific fields.

* Add change log

* Roll back unintended change to log4j2.xml
This commit is contained in:
Dan Torrey
2026-01-22 16:29:49 -06:00
committed by GitHub
parent f32b0324bc
commit 1f4bbcb41d
10 changed files with 233 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
changelog/unreleased/pr-24786.toml Normal file
View File

@@ -0,0 +1,4 @@
type = "a"
message = "Add new Full Message JSON field to the AWS Cloud Trail input to support custom pipeline parsing"
pulls = ["24786"]

16
graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java
View File

@@ -27,6 +27,7 @@ import org.graylog2.plugin.Message;
import org.graylog2.plugin.MessageFactory;
import org.graylog2.plugin.configuration.Configuration;
import org.graylog2.plugin.configuration.ConfigurationRequest;
import org.graylog2.plugin.configuration.fields.BooleanField;
import org.graylog2.plugin.inputs.annotations.ConfigClass;
import org.graylog2.plugin.inputs.annotations.FactoryClass;
import org.graylog2.plugin.inputs.codecs.AbstractCodec;
@@ -43,6 +44,7 @@ import static org.graylog.aws.inputs.cloudtrail.CloudTrailInput.Config.getOverri
public class CloudTrailCodec extends AbstractCodec {
public static final String NAME = "AWSCloudTrail";
public static final String CK_INCLUDE_FULL_MESSAGE_JSON = "include_full_message_json";
private final ObjectMapper objectMapper;
private final MessageFactory messageFactory;
@@ -65,6 +67,14 @@ public class CloudTrailCodec extends AbstractCodec {
message.addField("full_message", record.getFullMessage());
message.addField(AWS.SOURCE_GROUP_IDENTIFIER, true);
// Store full CloudTrail event as JSON if configured
if (configuration.getBoolean(CK_INCLUDE_FULL_MESSAGE_JSON)) {
final String fullMessageJson = record.getFullMessageJson(objectMapper);
if (fullMessageJson != null) {
message.addField("full_message_json", fullMessageJson);
}
}
// Apply override_source if configured
final String overrideSourceValue = configuration.getString(CK_OVERRIDE_SOURCE);
if (StringUtils.isNotBlank(overrideSourceValue)) {
@@ -98,6 +108,12 @@ public class CloudTrailCodec extends AbstractCodec {
public ConfigurationRequest getRequestedConfiguration() {
final ConfigurationRequest r = new ConfigurationRequest();
r.addField(getOverrideSourceFieldDefinition());
r.addField(new BooleanField(
CK_INCLUDE_FULL_MESSAGE_JSON,
"Include full_message_json?",
false,
"Store the complete CloudTrail event as JSON in the full_message_json field?"
));
return r;
}
}

2
graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/CloudTrailDriver.java
View File

@@ -19,6 +19,7 @@ package org.graylog.aws.inputs.cloudtrail.api;
import jakarta.inject.Inject;
import jakarta.ws.rs.BadRequestException;
import org.graylog.aws.config.AWSPluginConfiguration;
import org.graylog.aws.inputs.cloudtrail.CloudTrailCodec;
import org.graylog.aws.inputs.cloudtrail.CloudTrailInput;
import org.graylog.aws.inputs.cloudtrail.api.requests.CloudTrailCreateInputRequest;
import org.graylog.aws.inputs.cloudtrail.api.requests.CloudTrailRequest;
@@ -145,6 +146,7 @@ public class CloudTrailDriver {
configuration.put(CloudTrailInput.CK_POLLING_INTERVAL, request.pollingInterval());
configuration.put(CloudTrailInput.CK_OVERRIDE_SOURCE, request.overrideSource());
configuration.put(CloudTrailInput.CK_SQS_MESSAGE_BATCH_SIZE, request.sqsMessageBatchSize());
configuration.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, request.includeFullMessageJson());
final InputCreateRequest inputCreateRequest = InputCreateRequest.create(request.name(),
CloudTrailInput.TYPE,

7
graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/requests/CloudTrailCreateInputRequest.java
View File

@@ -32,6 +32,7 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest
private static final String POLLING_INTERVAL = "polling_interval";
private static final String OVERRIDE_SOURCE = "override_source";
private static final String SQS_MESSAGE_BATCH_SIZE = "sqs_message_batch_size";
private static final String INCLUDE_FULL_MESSAGE_JSON = "include_full_message_json";
@JsonProperty(NAME)
public abstract String name();
@@ -48,6 +49,9 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest
@JsonProperty(SQS_MESSAGE_BATCH_SIZE)
public abstract int sqsMessageBatchSize();
@JsonProperty(INCLUDE_FULL_MESSAGE_JSON)
public abstract boolean includeFullMessageJson();
@AutoValue.Builder
public static abstract class Builder implements CloudTrailRequest.Builder<Builder> {
@@ -71,6 +75,9 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest
@JsonProperty(SQS_MESSAGE_BATCH_SIZE)
public abstract Builder sqsMessageBatchSize(int sqsMessageBatchSize);
@JsonProperty(INCLUDE_FULL_MESSAGE_JSON)
public abstract Builder includeFullMessageJson(boolean includeFullMessageJson);
public abstract CloudTrailCreateInputRequest build();
}
}

17
graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/json/CloudTrailRecord.java
View File

@@ -17,6 +17,8 @@
package org.graylog.aws.inputs.cloudtrail.json;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps;
import java.io.Serializable;
@@ -116,4 +118,19 @@ public class CloudTrailRecord implements Serializable {
.orElse("<unknown user_name>");
}
/**
* Serializes the entire CloudTrail record as a JSON string for storage in the full_message_json field.
*
* @param objectMapper the ObjectMapper to use for JSON serialization
* @return JSON string of the entire CloudTrail record, or null if serialization fails
*/
public String getFullMessageJson(ObjectMapper objectMapper) {
try {
return objectMapper.writeValueAsString(this);
} catch (JsonProcessingException e) {
// If serialization fails, return null rather than throw
return null;
}
}
}

168
graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java
View File

@@ -16,6 +16,8 @@
*/
package org.graylog.aws.inputs.cloudtrail;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.MessageFactory;
import org.graylog2.plugin.TestMessageFactory;
@@ -29,8 +31,11 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -156,6 +161,169 @@ public class CloudTrailCodecTest {
assertEquals("123456789012", message.getField("session_issuer_user_account_id"));
}
@Test
public void testFullMessageJsonDisabledByDefault() {
final CloudTrailCodec codec = new CloudTrailCodec(Configuration.EMPTY_CONFIGURATION,
new ObjectMapperProvider().get(), messageFactory);
final RawMessage rawMessage = new RawMessage(("{\n" +
"\"eventVersion\": \"1.08\",\n" +
"\"userIdentity\": {\n" +
"\"type\": \"IAMUser\",\n" +
"\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" +
"\"arn\": \"arn:aws:iam::123456789012:user/Alice\",\n" +
"\"accountId\": \"123456789012\",\n" +
"\"userName\": \"Alice\"" +
"},\n" +
"\"eventTime\": \"2024-01-15T10:30:45Z\",\n" +
"\"eventSource\": \"s3.amazonaws.com\",\n" +
"\"eventName\": \"PutObject\",\n" +
"\"awsRegion\": \"us-east-1\",\n" +
"\"sourceIPAddress\": \"192.168.1.100\",\n" +
"\"userAgent\": \"aws-cli/2.0.0\",\n" +
"\"requestParameters\": {\n" +
"\"bucketName\": \"my-bucket\",\n" +
"\"key\": \"file.pdf\"\n" +
"},\n" +
"\"responseElements\": null,\n" +
"\"requestID\": \"ABC123\",\n" +
"\"eventID\": \"a1b2c3d4\",\n" +
"\"eventType\": \"AwsApiCall\",\n" +
"\"recipientAccountId\": \"123456789012\"\n" +
"}").getBytes(StandardCharsets.UTF_8));
Message message = codec.decodeSafe(rawMessage).get();
// full_message_json should not be present when disabled (default)
assertNull(message.getField("full_message_json"));
}
@Test
public void testFullMessageJsonEnabled() throws Exception {
final Map<String, Object> config = new HashMap<>();
config.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, true);
final Configuration configuration = new Configuration(config);
final CloudTrailCodec codec = new CloudTrailCodec(configuration,
new ObjectMapperProvider().get(), messageFactory);
final RawMessage rawMessage = new RawMessage(("{\n" +
"\"eventVersion\": \"1.08\",\n" +
"\"userIdentity\": {\n" +
"\"type\": \"IAMUser\",\n" +
"\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" +
"\"arn\": \"arn:aws:iam::123456789012:user/Alice\",\n" +
"\"accountId\": \"123456789012\",\n" +
"\"userName\": \"Alice\"" +
"},\n" +
"\"eventTime\": \"2024-01-15T10:30:45Z\",\n" +
"\"eventSource\": \"s3.amazonaws.com\",\n" +
"\"eventName\": \"PutObject\",\n" +
"\"awsRegion\": \"us-east-1\",\n" +
"\"sourceIPAddress\": \"192.168.1.100\",\n" +
"\"userAgent\": \"aws-cli/2.0.0\",\n" +
"\"requestParameters\": {\n" +
"\"bucketName\": \"my-bucket\",\n" +
"\"key\": \"file.pdf\"\n" +
"},\n" +
"\"responseElements\": null,\n" +
"\"requestID\": \"ABC123\",\n" +
"\"eventID\": \"a1b2c3d4\",\n" +
"\"eventType\": \"AwsApiCall\",\n" +
"\"recipientAccountId\": \"123456789012\"\n" +
"}").getBytes(StandardCharsets.UTF_8));
Message message = codec.decodeSafe(rawMessage).get();
// full_message_json should be present when enabled
assertNotNull(message.getField("full_message_json"));
// Parse and verify the JSON content
String fullMessageJson = message.getField("full_message_json").toString();
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(fullMessageJson);
// Verify key fields are present in the JSON
assertEquals("1.08", jsonNode.get("eventVersion").asText());
assertEquals("s3.amazonaws.com", jsonNode.get("eventSource").asText());
assertEquals("PutObject", jsonNode.get("eventName").asText());
assertEquals("us-east-1", jsonNode.get("awsRegion").asText());
assertEquals("192.168.1.100", jsonNode.get("sourceIPAddress").asText());
// Verify userIdentity is present and correct
assertNotNull(jsonNode.get("userIdentity"));
assertEquals("IAMUser", jsonNode.get("userIdentity").get("type").asText());
assertEquals("Alice", jsonNode.get("userIdentity").get("userName").asText());
// Verify requestParameters are present and correct
assertNotNull(jsonNode.get("requestParameters"));
assertEquals("my-bucket", jsonNode.get("requestParameters").get("bucketName").asText());
assertEquals("file.pdf", jsonNode.get("requestParameters").get("key").asText());
}
@Test
public void testFullMessageJsonWithNestedRequestParameters() throws Exception {
final Map<String, Object> config = new HashMap<>();
config.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, true);
final Configuration configuration = new Configuration(config);
final CloudTrailCodec codec = new CloudTrailCodec(configuration,
new ObjectMapperProvider().get(), messageFactory);
final RawMessage rawMessage = new RawMessage(("{\n" +
"\"eventVersion\": \"1.08\",\n" +
"\"userIdentity\": {\n" +
"\"type\": \"IAMUser\",\n" +
"\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" +
"\"arn\": \"arn:aws:iam::123456789012:user/SecurityAdmin\",\n" +
"\"accountId\": \"123456789012\",\n" +
"\"userName\": \"SecurityAdmin\"" +
"},\n" +
"\"eventTime\": \"2024-01-15T16:10:33Z\",\n" +
"\"eventSource\": \"s3.amazonaws.com\",\n" +
"\"eventName\": \"PutBucketPublicAccessBlock\",\n" +
"\"awsRegion\": \"us-east-1\",\n" +
"\"sourceIPAddress\": \"192.168.1.200\",\n" +
"\"userAgent\": \"console.amazonaws.com\",\n" +
"\"requestParameters\": {\n" +
"\"bucketName\": \"sensitive-data\",\n" +
"\"PublicAccessBlockConfiguration\": {\n" +
"\"BlockPublicAcls\": false,\n" +
"\"IgnorePublicAcls\": false,\n" +
"\"BlockPublicPolicy\": true,\n" +
"\"RestrictPublicBuckets\": true\n" +
"}\n" +
"},\n" +
"\"responseElements\": null,\n" +
"\"requestID\": \"JKL901MNO234\",\n" +
"\"eventID\": \"d4e5f6a7\",\n" +
"\"eventType\": \"AwsApiCall\",\n" +
"\"recipientAccountId\": \"123456789012\"\n" +
"}").getBytes(StandardCharsets.UTF_8));
Message message = codec.decodeSafe(rawMessage).get();
// full_message_json should be present
assertNotNull(message.getField("full_message_json"));
// Parse and verify nested requestParameters
String fullMessageJson = message.getField("full_message_json").toString();
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(fullMessageJson);
// Verify nested PublicAccessBlockConfiguration
JsonNode requestParams = jsonNode.get("requestParameters");
assertNotNull(requestParams);
assertEquals("sensitive-data", requestParams.get("bucketName").asText());
JsonNode publicAccessBlock = requestParams.get("PublicAccessBlockConfiguration");
assertNotNull(publicAccessBlock);
assertEquals(false, publicAccessBlock.get("BlockPublicAcls").asBoolean());
assertEquals(false, publicAccessBlock.get("IgnorePublicAcls").asBoolean());
assertEquals(true, publicAccessBlock.get("BlockPublicPolicy").asBoolean());
assertEquals(true, publicAccessBlock.get("RestrictPublicBuckets").asBoolean());
}
private RawMessage getRawMessageFromFile(String fileName) throws IOException, URISyntaxException {
File events = new File(this.getClass().getResource(fileName).toURI());
return new RawMessage(Files.readString(events.toPath(), StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8));

11
graylog2-web-interface/src/integrations/aws/cloudtrail/FormAdvancedOptions.tsx
View File

@@ -33,7 +33,7 @@ const FormAdvancedOptions = ({ onChange, handleSqsMessageBatchSizeChange }: Form
const { formData } = useContext(FormDataContext);
const { isAdvancedOptionsVisible, setAdvancedOptionsVisibility } = useContext(AdvancedOptionsContext);
const { overrideSource, awsCloudTrailThrottleEnabled, sqsMessageBatchSize } = formData;
const { overrideSource, awsCloudTrailThrottleEnabled, sqsMessageBatchSize, includeFullMessageJson } = formData;
const handleToggle = (visible) => {
setAdvancedOptionsVisibility(visible);
@@ -76,6 +76,15 @@ const FormAdvancedOptions = ({ onChange, handleSqsMessageBatchSizeChange }: Form
label="SQS Message Batch Size"
help="The maximum number of messages to query from SQS at a time. The maximum acceptable value is 10."
/>
<Input
id="includeFullMessageJson"
type="checkbox"
checked={includeFullMessageJson?.value}
onChange={onChange}
label="Include full_message_json?"
help="Store the complete CloudTrail event as JSON in the full_message_json field?"
/>
</AdditionalFields>
);
};

3
graylog2-web-interface/src/integrations/aws/cloudtrail/_initialFormData.js
View File

@@ -35,6 +35,9 @@ const DEFAULT_SETTINGS = {
sqsMessageBatchSize: {
value: 5,
},
includeFullMessageJson: {
value: false,
},
};
export default DEFAULT_SETTINGS;

4
graylog2-web-interface/src/integrations/aws/cloudtrail/common/formDataAdapter.ts
View File

@@ -34,6 +34,7 @@ export const toAWSCloudTrailInputCreateRequest = ({
key,
secret,
sqsMessageBatchSize,
includeFullMessageJson,
}: FormDataType): AWSCloudTrailInputCreateRequest => ({
name: awsCloudTrailName?.value,
...(awsAuthenticationType?.value === AWS_AUTH_TYPES.keysecret
@@ -53,6 +54,7 @@ export const toAWSCloudTrailInputCreateRequest = ({
assume_role_arn: awsAssumeRoleArn?.value,
override_source: overrideSource?.value,
sqs_message_batch_size: sqsMessageBatchSize?.value,
include_full_message_json: !!includeFullMessageJson?.value,
});
export const toGenericInputCreateRequest = ({
@@ -70,6 +72,7 @@ export const toGenericInputCreateRequest = ({
secret,
overrideSource,
sqsMessageBatchSize,
includeFullMessageJson,
}: FormDataType): AWSCloudTrailGenericInputCreateRequest => ({
type: 'org.graylog.aws.inputs.cloudtrail.CloudTrailInput',
title: awsCloudTrailName?.value,
@@ -92,5 +95,6 @@ export const toGenericInputCreateRequest = ({
assume_role_arn: awsAssumeRoleArn?.value,
override_source: overrideSource?.value,
sqs_message_batch_size: sqsMessageBatchSize?.value,
include_full_message_json: !!includeFullMessageJson?.value,
},
});

2
graylog2-web-interface/src/integrations/aws/cloudtrail/types.ts
View File

@@ -31,6 +31,7 @@ export type AWSCloudTrailGenericInputCreateRequest = {
assume_role_arn: string;
override_source?: string;
sqs_message_batch_size: number;
include_full_message_json: boolean;
};
};
@@ -46,6 +47,7 @@ export type AWSCloudTrailInputCreateRequest = {
assume_role_arn: string;
override_source?: string;
sqs_message_batch_size: number;
include_full_message_json: boolean;
};
export type ErrorMessageType = {