diff --git a/changelog/unreleased/pr-24786.toml b/changelog/unreleased/pr-24786.toml new file mode 100644 index 0000000000..35e53e0e97 --- /dev/null +++ b/changelog/unreleased/pr-24786.toml @@ -0,0 +1,4 @@ +type = "a" +message = "Add new Full Message JSON field to the AWS Cloud Trail input to support custom pipeline parsing" + +pulls = ["24786"] diff --git a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java index 0096c7c3e0..daaf417e3b 100644 --- a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java +++ b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodec.java @@ -27,6 +27,7 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.configuration.Configuration; import org.graylog2.plugin.configuration.ConfigurationRequest; +import org.graylog2.plugin.configuration.fields.BooleanField; import org.graylog2.plugin.inputs.annotations.ConfigClass; import org.graylog2.plugin.inputs.annotations.FactoryClass; import org.graylog2.plugin.inputs.codecs.AbstractCodec; @@ -43,6 +44,7 @@ import static org.graylog.aws.inputs.cloudtrail.CloudTrailInput.Config.getOverri public class CloudTrailCodec extends AbstractCodec { public static final String NAME = "AWSCloudTrail"; + public static final String CK_INCLUDE_FULL_MESSAGE_JSON = "include_full_message_json"; private final ObjectMapper objectMapper; private final MessageFactory messageFactory; @@ -65,6 +67,14 @@ public class CloudTrailCodec extends AbstractCodec { message.addField("full_message", record.getFullMessage()); message.addField(AWS.SOURCE_GROUP_IDENTIFIER, true); + // Store full CloudTrail event as JSON if configured + if (configuration.getBoolean(CK_INCLUDE_FULL_MESSAGE_JSON)) { + final String fullMessageJson = record.getFullMessageJson(objectMapper); + if (fullMessageJson != null) { + message.addField("full_message_json", fullMessageJson); + } + } + // Apply override_source if configured final String overrideSourceValue = configuration.getString(CK_OVERRIDE_SOURCE); if (StringUtils.isNotBlank(overrideSourceValue)) { @@ -98,6 +108,12 @@ public class CloudTrailCodec extends AbstractCodec { public ConfigurationRequest getRequestedConfiguration() { final ConfigurationRequest r = new ConfigurationRequest(); r.addField(getOverrideSourceFieldDefinition()); + r.addField(new BooleanField( + CK_INCLUDE_FULL_MESSAGE_JSON, + "Include full_message_json?", + false, + "Store the complete CloudTrail event as JSON in the full_message_json field?" + )); return r; } } diff --git a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/CloudTrailDriver.java b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/CloudTrailDriver.java index 65e4856f4e..7d5caa2fc8 100644 --- a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/CloudTrailDriver.java +++ b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/CloudTrailDriver.java @@ -19,6 +19,7 @@ package org.graylog.aws.inputs.cloudtrail.api; import jakarta.inject.Inject; import jakarta.ws.rs.BadRequestException; import org.graylog.aws.config.AWSPluginConfiguration; +import org.graylog.aws.inputs.cloudtrail.CloudTrailCodec; import org.graylog.aws.inputs.cloudtrail.CloudTrailInput; import org.graylog.aws.inputs.cloudtrail.api.requests.CloudTrailCreateInputRequest; import org.graylog.aws.inputs.cloudtrail.api.requests.CloudTrailRequest; @@ -145,6 +146,7 @@ public class CloudTrailDriver { configuration.put(CloudTrailInput.CK_POLLING_INTERVAL, request.pollingInterval()); configuration.put(CloudTrailInput.CK_OVERRIDE_SOURCE, request.overrideSource()); configuration.put(CloudTrailInput.CK_SQS_MESSAGE_BATCH_SIZE, request.sqsMessageBatchSize()); + configuration.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, request.includeFullMessageJson()); final InputCreateRequest inputCreateRequest = InputCreateRequest.create(request.name(), CloudTrailInput.TYPE, diff --git a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/requests/CloudTrailCreateInputRequest.java b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/requests/CloudTrailCreateInputRequest.java index 64af43adcc..6911693293 100644 --- a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/requests/CloudTrailCreateInputRequest.java +++ b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/api/requests/CloudTrailCreateInputRequest.java @@ -32,6 +32,7 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest private static final String POLLING_INTERVAL = "polling_interval"; private static final String OVERRIDE_SOURCE = "override_source"; private static final String SQS_MESSAGE_BATCH_SIZE = "sqs_message_batch_size"; + private static final String INCLUDE_FULL_MESSAGE_JSON = "include_full_message_json"; @JsonProperty(NAME) public abstract String name(); @@ -48,6 +49,9 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest @JsonProperty(SQS_MESSAGE_BATCH_SIZE) public abstract int sqsMessageBatchSize(); + @JsonProperty(INCLUDE_FULL_MESSAGE_JSON) + public abstract boolean includeFullMessageJson(); + @AutoValue.Builder public static abstract class Builder implements CloudTrailRequest.Builder { @@ -71,6 +75,9 @@ public abstract class CloudTrailCreateInputRequest implements CloudTrailRequest @JsonProperty(SQS_MESSAGE_BATCH_SIZE) public abstract Builder sqsMessageBatchSize(int sqsMessageBatchSize); + @JsonProperty(INCLUDE_FULL_MESSAGE_JSON) + public abstract Builder includeFullMessageJson(boolean includeFullMessageJson); + public abstract CloudTrailCreateInputRequest build(); } } diff --git a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/json/CloudTrailRecord.java b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/json/CloudTrailRecord.java index fdca003feb..88364992ac 100644 --- a/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/json/CloudTrailRecord.java +++ b/graylog2-server/src/main/java/org/graylog/aws/inputs/cloudtrail/json/CloudTrailRecord.java @@ -17,6 +17,8 @@ package org.graylog.aws.inputs.cloudtrail.json; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import java.io.Serializable; @@ -116,4 +118,19 @@ public class CloudTrailRecord implements Serializable { .orElse(""); } + /** + * Serializes the entire CloudTrail record as a JSON string for storage in the full_message_json field. + * + * @param objectMapper the ObjectMapper to use for JSON serialization + * @return JSON string of the entire CloudTrail record, or null if serialization fails + */ + public String getFullMessageJson(ObjectMapper objectMapper) { + try { + return objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + // If serialization fails, return null rather than throw + return null; + } + } + } diff --git a/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java b/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java index 5b4f317688..5a8edc4cb0 100644 --- a/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java +++ b/graylog2-server/src/test/java/org/graylog/aws/inputs/cloudtrail/CloudTrailCodecTest.java @@ -16,6 +16,8 @@ */ package org.graylog.aws.inputs.cloudtrail; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageFactory; import org.graylog2.plugin.TestMessageFactory; @@ -29,8 +31,11 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -156,6 +161,169 @@ public class CloudTrailCodecTest { assertEquals("123456789012", message.getField("session_issuer_user_account_id")); } + @Test + public void testFullMessageJsonDisabledByDefault() { + final CloudTrailCodec codec = new CloudTrailCodec(Configuration.EMPTY_CONFIGURATION, + new ObjectMapperProvider().get(), messageFactory); + + final RawMessage rawMessage = new RawMessage(("{\n" + + "\"eventVersion\": \"1.08\",\n" + + "\"userIdentity\": {\n" + + "\"type\": \"IAMUser\",\n" + + "\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" + + "\"arn\": \"arn:aws:iam::123456789012:user/Alice\",\n" + + "\"accountId\": \"123456789012\",\n" + + "\"userName\": \"Alice\"" + + "},\n" + + "\"eventTime\": \"2024-01-15T10:30:45Z\",\n" + + "\"eventSource\": \"s3.amazonaws.com\",\n" + + "\"eventName\": \"PutObject\",\n" + + "\"awsRegion\": \"us-east-1\",\n" + + "\"sourceIPAddress\": \"192.168.1.100\",\n" + + "\"userAgent\": \"aws-cli/2.0.0\",\n" + + "\"requestParameters\": {\n" + + "\"bucketName\": \"my-bucket\",\n" + + "\"key\": \"file.pdf\"\n" + + "},\n" + + "\"responseElements\": null,\n" + + "\"requestID\": \"ABC123\",\n" + + "\"eventID\": \"a1b2c3d4\",\n" + + "\"eventType\": \"AwsApiCall\",\n" + + "\"recipientAccountId\": \"123456789012\"\n" + + "}").getBytes(StandardCharsets.UTF_8)); + + Message message = codec.decodeSafe(rawMessage).get(); + + // full_message_json should not be present when disabled (default) + assertNull(message.getField("full_message_json")); + } + + @Test + public void testFullMessageJsonEnabled() throws Exception { + final Map config = new HashMap<>(); + config.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, true); + final Configuration configuration = new Configuration(config); + + final CloudTrailCodec codec = new CloudTrailCodec(configuration, + new ObjectMapperProvider().get(), messageFactory); + + final RawMessage rawMessage = new RawMessage(("{\n" + + "\"eventVersion\": \"1.08\",\n" + + "\"userIdentity\": {\n" + + "\"type\": \"IAMUser\",\n" + + "\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" + + "\"arn\": \"arn:aws:iam::123456789012:user/Alice\",\n" + + "\"accountId\": \"123456789012\",\n" + + "\"userName\": \"Alice\"" + + "},\n" + + "\"eventTime\": \"2024-01-15T10:30:45Z\",\n" + + "\"eventSource\": \"s3.amazonaws.com\",\n" + + "\"eventName\": \"PutObject\",\n" + + "\"awsRegion\": \"us-east-1\",\n" + + "\"sourceIPAddress\": \"192.168.1.100\",\n" + + "\"userAgent\": \"aws-cli/2.0.0\",\n" + + "\"requestParameters\": {\n" + + "\"bucketName\": \"my-bucket\",\n" + + "\"key\": \"file.pdf\"\n" + + "},\n" + + "\"responseElements\": null,\n" + + "\"requestID\": \"ABC123\",\n" + + "\"eventID\": \"a1b2c3d4\",\n" + + "\"eventType\": \"AwsApiCall\",\n" + + "\"recipientAccountId\": \"123456789012\"\n" + + "}").getBytes(StandardCharsets.UTF_8)); + + Message message = codec.decodeSafe(rawMessage).get(); + + // full_message_json should be present when enabled + assertNotNull(message.getField("full_message_json")); + + // Parse and verify the JSON content + String fullMessageJson = message.getField("full_message_json").toString(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(fullMessageJson); + + // Verify key fields are present in the JSON + assertEquals("1.08", jsonNode.get("eventVersion").asText()); + assertEquals("s3.amazonaws.com", jsonNode.get("eventSource").asText()); + assertEquals("PutObject", jsonNode.get("eventName").asText()); + assertEquals("us-east-1", jsonNode.get("awsRegion").asText()); + assertEquals("192.168.1.100", jsonNode.get("sourceIPAddress").asText()); + + // Verify userIdentity is present and correct + assertNotNull(jsonNode.get("userIdentity")); + assertEquals("IAMUser", jsonNode.get("userIdentity").get("type").asText()); + assertEquals("Alice", jsonNode.get("userIdentity").get("userName").asText()); + + // Verify requestParameters are present and correct + assertNotNull(jsonNode.get("requestParameters")); + assertEquals("my-bucket", jsonNode.get("requestParameters").get("bucketName").asText()); + assertEquals("file.pdf", jsonNode.get("requestParameters").get("key").asText()); + } + + @Test + public void testFullMessageJsonWithNestedRequestParameters() throws Exception { + final Map config = new HashMap<>(); + config.put(CloudTrailCodec.CK_INCLUDE_FULL_MESSAGE_JSON, true); + final Configuration configuration = new Configuration(config); + + final CloudTrailCodec codec = new CloudTrailCodec(configuration, + new ObjectMapperProvider().get(), messageFactory); + + final RawMessage rawMessage = new RawMessage(("{\n" + + "\"eventVersion\": \"1.08\",\n" + + "\"userIdentity\": {\n" + + "\"type\": \"IAMUser\",\n" + + "\"principalId\": \"AIDAJ45Q7YFFAREXAMPLE\",\n" + + "\"arn\": \"arn:aws:iam::123456789012:user/SecurityAdmin\",\n" + + "\"accountId\": \"123456789012\",\n" + + "\"userName\": \"SecurityAdmin\"" + + "},\n" + + "\"eventTime\": \"2024-01-15T16:10:33Z\",\n" + + "\"eventSource\": \"s3.amazonaws.com\",\n" + + "\"eventName\": \"PutBucketPublicAccessBlock\",\n" + + "\"awsRegion\": \"us-east-1\",\n" + + "\"sourceIPAddress\": \"192.168.1.200\",\n" + + "\"userAgent\": \"console.amazonaws.com\",\n" + + "\"requestParameters\": {\n" + + "\"bucketName\": \"sensitive-data\",\n" + + "\"PublicAccessBlockConfiguration\": {\n" + + "\"BlockPublicAcls\": false,\n" + + "\"IgnorePublicAcls\": false,\n" + + "\"BlockPublicPolicy\": true,\n" + + "\"RestrictPublicBuckets\": true\n" + + "}\n" + + "},\n" + + "\"responseElements\": null,\n" + + "\"requestID\": \"JKL901MNO234\",\n" + + "\"eventID\": \"d4e5f6a7\",\n" + + "\"eventType\": \"AwsApiCall\",\n" + + "\"recipientAccountId\": \"123456789012\"\n" + + "}").getBytes(StandardCharsets.UTF_8)); + + Message message = codec.decodeSafe(rawMessage).get(); + + // full_message_json should be present + assertNotNull(message.getField("full_message_json")); + + // Parse and verify nested requestParameters + String fullMessageJson = message.getField("full_message_json").toString(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(fullMessageJson); + + // Verify nested PublicAccessBlockConfiguration + JsonNode requestParams = jsonNode.get("requestParameters"); + assertNotNull(requestParams); + assertEquals("sensitive-data", requestParams.get("bucketName").asText()); + + JsonNode publicAccessBlock = requestParams.get("PublicAccessBlockConfiguration"); + assertNotNull(publicAccessBlock); + assertEquals(false, publicAccessBlock.get("BlockPublicAcls").asBoolean()); + assertEquals(false, publicAccessBlock.get("IgnorePublicAcls").asBoolean()); + assertEquals(true, publicAccessBlock.get("BlockPublicPolicy").asBoolean()); + assertEquals(true, publicAccessBlock.get("RestrictPublicBuckets").asBoolean()); + } + private RawMessage getRawMessageFromFile(String fileName) throws IOException, URISyntaxException { File events = new File(this.getClass().getResource(fileName).toURI()); return new RawMessage(Files.readString(events.toPath(), StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8)); diff --git a/graylog2-web-interface/src/integrations/aws/cloudtrail/FormAdvancedOptions.tsx b/graylog2-web-interface/src/integrations/aws/cloudtrail/FormAdvancedOptions.tsx index b9721a5977..8560a4868c 100644 --- a/graylog2-web-interface/src/integrations/aws/cloudtrail/FormAdvancedOptions.tsx +++ b/graylog2-web-interface/src/integrations/aws/cloudtrail/FormAdvancedOptions.tsx @@ -33,7 +33,7 @@ const FormAdvancedOptions = ({ onChange, handleSqsMessageBatchSizeChange }: Form const { formData } = useContext(FormDataContext); const { isAdvancedOptionsVisible, setAdvancedOptionsVisibility } = useContext(AdvancedOptionsContext); - const { overrideSource, awsCloudTrailThrottleEnabled, sqsMessageBatchSize } = formData; + const { overrideSource, awsCloudTrailThrottleEnabled, sqsMessageBatchSize, includeFullMessageJson } = formData; const handleToggle = (visible) => { setAdvancedOptionsVisibility(visible); @@ -76,6 +76,15 @@ const FormAdvancedOptions = ({ onChange, handleSqsMessageBatchSizeChange }: Form label="SQS Message Batch Size" help="The maximum number of messages to query from SQS at a time. The maximum acceptable value is 10." /> + + ); }; diff --git a/graylog2-web-interface/src/integrations/aws/cloudtrail/_initialFormData.js b/graylog2-web-interface/src/integrations/aws/cloudtrail/_initialFormData.js index b8eb562a96..7bb78f7ac4 100644 --- a/graylog2-web-interface/src/integrations/aws/cloudtrail/_initialFormData.js +++ b/graylog2-web-interface/src/integrations/aws/cloudtrail/_initialFormData.js @@ -35,6 +35,9 @@ const DEFAULT_SETTINGS = { sqsMessageBatchSize: { value: 5, }, + includeFullMessageJson: { + value: false, + }, }; export default DEFAULT_SETTINGS; diff --git a/graylog2-web-interface/src/integrations/aws/cloudtrail/common/formDataAdapter.ts b/graylog2-web-interface/src/integrations/aws/cloudtrail/common/formDataAdapter.ts index ced671be47..940dd3817f 100644 --- a/graylog2-web-interface/src/integrations/aws/cloudtrail/common/formDataAdapter.ts +++ b/graylog2-web-interface/src/integrations/aws/cloudtrail/common/formDataAdapter.ts @@ -34,6 +34,7 @@ export const toAWSCloudTrailInputCreateRequest = ({ key, secret, sqsMessageBatchSize, + includeFullMessageJson, }: FormDataType): AWSCloudTrailInputCreateRequest => ({ name: awsCloudTrailName?.value, ...(awsAuthenticationType?.value === AWS_AUTH_TYPES.keysecret @@ -53,6 +54,7 @@ export const toAWSCloudTrailInputCreateRequest = ({ assume_role_arn: awsAssumeRoleArn?.value, override_source: overrideSource?.value, sqs_message_batch_size: sqsMessageBatchSize?.value, + include_full_message_json: !!includeFullMessageJson?.value, }); export const toGenericInputCreateRequest = ({ @@ -70,6 +72,7 @@ export const toGenericInputCreateRequest = ({ secret, overrideSource, sqsMessageBatchSize, + includeFullMessageJson, }: FormDataType): AWSCloudTrailGenericInputCreateRequest => ({ type: 'org.graylog.aws.inputs.cloudtrail.CloudTrailInput', title: awsCloudTrailName?.value, @@ -92,5 +95,6 @@ export const toGenericInputCreateRequest = ({ assume_role_arn: awsAssumeRoleArn?.value, override_source: overrideSource?.value, sqs_message_batch_size: sqsMessageBatchSize?.value, + include_full_message_json: !!includeFullMessageJson?.value, }, }); diff --git a/graylog2-web-interface/src/integrations/aws/cloudtrail/types.ts b/graylog2-web-interface/src/integrations/aws/cloudtrail/types.ts index b4caa6b82a..7e42f129c3 100644 --- a/graylog2-web-interface/src/integrations/aws/cloudtrail/types.ts +++ b/graylog2-web-interface/src/integrations/aws/cloudtrail/types.ts @@ -31,6 +31,7 @@ export type AWSCloudTrailGenericInputCreateRequest = { assume_role_arn: string; override_source?: string; sqs_message_batch_size: number; + include_full_message_json: boolean; }; }; @@ -46,6 +47,7 @@ export type AWSCloudTrailInputCreateRequest = { assume_role_arn: string; override_source?: string; sqs_message_batch_size: number; + include_full_message_json: boolean; }; export type ErrorMessageType = {