Adapt DatanodeClusterIT for full cert setup of all nodes (#16245)

* Adapt DatanodeClusterIT for full cert setup of all nodes

* generated passwords for DatanodeSecuritySetupIT, better truststore

* datanode integration tests code cleanup

* code cleanup
This commit is contained in:
Tomas Dvorak
2023-08-24 12:44:03 +02:00
committed by GitHub
parent eb9e30c9a2
commit 716f71cf46
7 changed files with 253 additions and 121 deletions

View File

@@ -16,59 +16,141 @@
*/
package org.graylog.datanode.integration;
import com.github.rholder.retry.Attempt;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.RetryListener;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import io.restassured.RestAssured;
import io.restassured.response.ValidatableResponse;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.http.NoHttpResponseException;
import org.graylog.datanode.configuration.variants.KeystoreInformation;
import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
import org.graylog.datanode.testinfra.DatanodeDockerHooksAdapter;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.SocketException;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.graylog.datanode.testinfra.DatanodeContainerizedBackend.IMAGE_WORKING_DIR;
public class DatanodeClusterIT {
private DatanodeContainerizedBackend primaryNode;
private DatanodeContainerizedBackend secondaryNode;
private static final Logger LOG = LoggerFactory.getLogger(DatanodeClusterIT.class);
@BeforeEach
void setUp() {
primaryNode = new DatanodeContainerizedBackend("nodeA", datanodeContainer -> datanodeContainer
.withEnv("GRAYLOG_DATANODE_CLUSTER_INITIAL_MANAGER_NODES", "nodeA")
.withEnv("GRAYLOG_DATANODE_OPENSEARCH_DISCOVERY_SEED_HOSTS", "nodeA:9300")
).start();
private DatanodeContainerizedBackend nodeA;
private DatanodeContainerizedBackend nodeB;
secondaryNode = new DatanodeContainerizedBackend(
primaryNode.getNetwork(),
primaryNode.getMongodbContainer(),
"nodeB",
datanodeContainer ->
datanodeContainer
.withEnv("GRAYLOG_DATANODE_CLUSTER_INITIAL_MANAGER_NODES", "nodeA")
.withEnv("GRAYLOG_DATANODE_OPENSEARCH_DISCOVERY_SEED_HOSTS", "nodeA:9300")
@TempDir
static Path tempDir;
private String hostnameNodeA;
private KeyStore trustStore;
private String usernameNodeA;
private String passwordNodeA;
@BeforeEach
void setUp() throws GeneralSecurityException, IOException {
// first generate a self-signed CA
final KeystoreInformation ca = DatanodeSecurityTestUtils.generateCa(tempDir);
trustStore = DatanodeSecurityTestUtils.buildTruststore(ca);
usernameNodeA = RandomStringUtils.randomAlphabetic(10);
passwordNodeA = RandomStringUtils.randomAlphabetic(10);
final String usernameNodeB = RandomStringUtils.randomAlphabetic(10);
final String passwordNodeB = RandomStringUtils.randomAlphabetic(10);
hostnameNodeA = "graylog-datanode-host-" + RandomStringUtils.random(8, "0123456789abcdef");
final KeystoreInformation transportNodeA = DatanodeSecurityTestUtils.generateTransportCert(tempDir, ca, "nodeA");
final KeystoreInformation httpNodeA = DatanodeSecurityTestUtils.generateHttpCert(tempDir, ca, "nodeA");
final String hostnameNodeB = "graylog-datanode-host-" + RandomStringUtils.random(8, "0123456789abcdef");
final KeystoreInformation transportNodeB = DatanodeSecurityTestUtils.generateTransportCert(tempDir, ca, "nodeB");
final KeystoreInformation httpNodeB = DatanodeSecurityTestUtils.generateHttpCert(tempDir, ca, "nodeB");
nodeA = new DatanodeContainerizedBackend(hostnameNodeA, datanodeContainer -> {
datanodeContainer.withEnv("GRAYLOG_DATANODE_CLUSTER_INITIAL_MANAGER_NODES", hostnameNodeA);
datanodeContainer.withEnv("GRAYLOG_DATANODE_OPENSEARCH_DISCOVERY_SEED_HOSTS", hostnameNodeA + ":9300");
datanodeContainer.withFileSystemBind(transportNodeA.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-transport-certificates.p12");
datanodeContainer.withFileSystemBind(httpNodeA.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-https-certificates.p12");
// configure transport security
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE", "datanode-transport-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE_PASSWORD", transportNodeA.passwordAsString());
datanodeContainer.withEnv("GRAYLOG_DATANODE_INSECURE_STARTUP", "false");
// configure http security
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE", "datanode-https-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE_PASSWORD", httpNodeA.passwordAsString());
// configure initial admin username and password for Opensearch REST
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_USERNAME", usernameNodeA);
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_PASSWORD", passwordNodeA);
// this is the interface that we bind opensearch to. It must be 0.0.0.0 if we want
// to be able to reach opensearch from outside the container and docker network (true?)
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_BIND_ADDRESS", "0.0.0.0");
// HOSTNAME is used to generate the SSL certificates and to communicate inside the
// container and docker network, where we do the hostname validation.
datanodeContainer.withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName(hostnameNodeA));
datanodeContainer.withEnv("GRAYLOG_DATANODE_HOSTNAME", hostnameNodeA);
}).start();
nodeB = new DatanodeContainerizedBackend(
nodeA.getNetwork(),
nodeA.getMongodbContainer(),
hostnameNodeB,
datanodeContainer -> {
datanodeContainer.withEnv("GRAYLOG_DATANODE_CLUSTER_INITIAL_MANAGER_NODES", hostnameNodeA);
datanodeContainer.withEnv("GRAYLOG_DATANODE_OPENSEARCH_DISCOVERY_SEED_HOSTS", hostnameNodeA + ":9300");
datanodeContainer.withFileSystemBind(transportNodeB.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-transport-certificates.p12");
datanodeContainer.withFileSystemBind(httpNodeB.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-https-certificates.p12");
// configure transport security
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE", "datanode-transport-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE_PASSWORD", transportNodeB.passwordAsString());
datanodeContainer.withEnv("GRAYLOG_DATANODE_INSECURE_STARTUP", "false");
// configure http security
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE", "datanode-https-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE_PASSWORD", httpNodeB.passwordAsString());
// configure initial admin username and password for Opensearch REST
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_USERNAME", usernameNodeB);
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_PASSWORD", passwordNodeB);
// this is the interface that we bind opensearch to. It must be 0.0.0.0 if we want
// to be able to reach opensearch from outside the container and docker network (true?)
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_BIND_ADDRESS", "0.0.0.0");
// HOSTNAME is used to generate the SSL certificates and to communicate inside the
// container and docker network, where we do the hostname validation.
datanodeContainer.withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName(hostnameNodeB));
datanodeContainer.withEnv("GRAYLOG_DATANODE_HOSTNAME", hostnameNodeB);
}
);
secondaryNode.start();
nodeB.start();
}
@AfterEach
void tearDown() {
secondaryNode.stop();
primaryNode.stop();
nodeB.stop();
nodeA.stop();
}
@Test
@@ -81,9 +163,10 @@ public class DatanodeClusterIT {
.retryIfException(input -> input instanceof NoHttpResponseException)
.retryIfException(input -> input instanceof SocketException)
.retryIfResult(input -> !input.extract().body().path("number_of_nodes").equals(2))
.retryIfResult(input -> !input.extract().body().path("status").equals("green"))
.build();
final Integer opensearchPort = primaryNode.getOpensearchRestPort();
final Integer opensearchPort = nodeA.getOpensearchRestPort();
retryer.call(() -> this.getStatus(opensearchPort))
.assertThat()
@@ -91,14 +174,16 @@ public class DatanodeClusterIT {
.body("number_of_nodes", Matchers.equalTo(2))
.body("discovered_cluster_manager", Matchers.equalTo(true));
} catch (RetryException retryException) {
LOG.error("DataNode Container logs follow:\n" + primaryNode.getLogs());
LOG.error("DataNode Container logs follow:\n" + nodeA.getLogs());
throw retryException;
}
}
private ValidatableResponse getStatus(Integer mappedPort) {
return RestAssured.given()
.get("http://localhost:" + mappedPort + "/_cluster/health")
.trustStore(trustStore)
.auth().basic(usernameNodeA, passwordNodeA)
.get("https://localhost:" + mappedPort + "/_cluster/health")
.then();
}
}

View File

@@ -25,11 +25,8 @@ import io.restassured.RestAssured;
import io.restassured.response.ValidatableResponse;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.http.NoHttpResponseException;
import org.graylog.datanode.configuration.variants.KeystoreInformation;
import org.graylog.datanode.testinfra.DatanodeContainerizedBackend;
import org.graylog.security.certutil.CertutilCa;
import org.graylog.security.certutil.CertutilCert;
import org.graylog.security.certutil.CertutilHttp;
import org.graylog.security.certutil.console.TestableConsole;
import org.graylog2.plugin.Tools;
import org.graylog2.shared.utilities.StringUtils;
import org.hamcrest.Matchers;
@@ -39,16 +36,11 @@ import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.SocketException;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -58,43 +50,48 @@ import static org.graylog.datanode.testinfra.DatanodeContainerizedBackend.IMAGE_
public class DatanodeSecuritySetupIT {
private static final Logger LOG = LoggerFactory.getLogger(DatanodeSecuritySetupIT.class);
public static final String OPENSEARCH_REST_USERNAME = "admin";
public static final String OPENSEARCH_REST_PASSWORD = "admin";
@TempDir
static Path tempDir;
private DatanodeContainerizedBackend backend;
private Path httpCert;
private String httpUsername;
private String httpPassword;
private KeyStore trustStore;
private String containerHostname;
@BeforeEach
void setUp() {
void setUp() throws IOException, GeneralSecurityException {
String containerHostname = "graylog-datanode-host-" + RandomStringUtils.random(8, "0123456789abcdef");
containerHostname = "graylog-datanode-host-" + RandomStringUtils.random(8, "0123456789abcdef");
// first generate a self-signed CA
final Path ca = generateCa();
KeystoreInformation ca = DatanodeSecurityTestUtils.generateCa(tempDir);
trustStore = DatanodeSecurityTestUtils.buildTruststore(ca);
// use the CA to generate transport certificate keystore
final Path nodeCert = generateNodeCert(ca);
final KeystoreInformation transportCert = DatanodeSecurityTestUtils.generateTransportCert(tempDir, ca, containerHostname);
// use the CA to generate HTTP certificate keystore
httpCert = generateHttpCert(ca, containerHostname, Tools.getLocalCanonicalHostname());
final KeystoreInformation httpCert = DatanodeSecurityTestUtils.generateHttpCert(tempDir, ca, containerHostname, Tools.getLocalCanonicalHostname());
backend = new DatanodeContainerizedBackend(datanodeContainer -> {
httpUsername = RandomStringUtils.randomAlphabetic(10);
httpPassword = RandomStringUtils.randomAlphabetic(10);
backend = new DatanodeContainerizedBackend(containerHostname, datanodeContainer -> {
// provide the keystore files to the docker container
datanodeContainer.withFileSystemBind(nodeCert.toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-transport-certificates.p12");
datanodeContainer.withFileSystemBind(httpCert.toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-https-certificates.p12");
datanodeContainer.withFileSystemBind(transportCert.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-transport-certificates.p12");
datanodeContainer.withFileSystemBind(httpCert.location().toAbsolutePath().toString(), IMAGE_WORKING_DIR + "/bin/config/datanode-https-certificates.p12");
// configure transport security
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE", "datanode-transport-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE_PASSWORD", "password");
datanodeContainer.withEnv("GRAYLOG_DATANODE_TRANSPORT_CERTIFICATE_PASSWORD", transportCert.passwordAsString());
datanodeContainer.withEnv("GRAYLOG_DATANODE_INSECURE_STARTUP", "false");
// configure http security
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE", "datanode-https-certificates.p12");
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE_PASSWORD", "password");
datanodeContainer.withEnv("GRAYLOG_DATANODE_HTTP_CERTIFICATE_PASSWORD", httpCert.passwordAsString());
// configure initial admin username and password for Opensearch REST
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_USERNAME", OPENSEARCH_REST_USERNAME);
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_PASSWORD", OPENSEARCH_REST_PASSWORD);
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_USERNAME", httpUsername);
datanodeContainer.withEnv("GRAYLOG_DATANODE_REST_API_PASSWORD", httpPassword);
// this is the interface that we bind opensearch to. It must be 0.0.0.0 if we want
// to be able to reach opensearch from outside the container and docker network (true?)
@@ -104,23 +101,23 @@ public class DatanodeSecuritySetupIT {
// container and docker network, where we do the hostname validation.
datanodeContainer.withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName(containerHostname));
datanodeContainer.withEnv("GRAYLOG_DATANODE_HOSTNAME", containerHostname);
datanodeContainer.withEnv("GRAYLOG_DATANODE_SINGLE_NODE_ONLY", "true");
}).start();
}
@Test
void testSecuredSetup() throws ExecutionException, RetryException, CertificateException, KeyStoreException, IOException, NoSuchAlgorithmException {
final KeyStore trustStore = buildTruststore(httpCert, "password");
void testSecuredSetup() throws ExecutionException, RetryException {
waitForOpensearchAvailableStatus(backend.getDatanodeRestPort(), trustStore);
try {
given()
.auth().basic(OPENSEARCH_REST_USERNAME, OPENSEARCH_REST_PASSWORD)
.auth().basic(httpUsername, httpPassword)
.trustStore(trustStore)
.get("https://localhost:" + backend.getOpensearchRestPort())
.then().assertThat()
.body("name", Matchers.equalTo("node1"))
.body("name", Matchers.equalTo(containerHostname))
.body("cluster_name", Matchers.equalTo("datanode-cluster"));
} catch (Exception ex) {
LOG.error("Error connecting to OpenSearch in the DataNode, showing logs:\n{}", backend.getLogs());
@@ -150,66 +147,4 @@ public class DatanodeSecuritySetupIT {
throw ex;
}
}
/**
* Take the HTTP keystore and extract CA certificate chain as a truststore.
*/
private KeyStore buildTruststore(Path keystorePath, String password) throws KeyStoreException, CertificateException, IOException, NoSuchAlgorithmException {
try (FileInputStream fis = new FileInputStream(keystorePath.toFile())) {
KeyStore keyStore = KeyStore.getInstance("PKCS12");
keyStore.load(fis, password.toCharArray());
KeyStore trustStore = KeyStore.getInstance("PKCS12");
trustStore.load(null, null);
final Certificate[] certs = keyStore.getCertificateChain("datanode");
for (Certificate cert : certs) {
if (cert instanceof final X509Certificate x509Certificate) {
final String alias = x509Certificate.getSubjectX500Principal().getName();
trustStore.setCertificateEntry(alias, x509Certificate);
}
}
return trustStore;
}
}
private Path generateCa() {
final Path certPath = tempDir.resolve("test-ca.p12");
final TestableConsole input = TestableConsole.empty().silent()
.register("Enter CA password", "password");
final CertutilCa command = new CertutilCa(certPath.toAbsolutePath().toString(), input);
command.run();
return certPath;
}
private Path generateNodeCert(Path caPath) {
final Path nodePath = tempDir.resolve("test-node.p12");
TestableConsole inputCert = TestableConsole.empty().silent()
.register("Enter CA password", "password")
.register("Enter datanode certificate password", "password");
CertutilCert certutilCert = new CertutilCert(
caPath.toAbsolutePath().toString(),
nodePath.toAbsolutePath().toString(),
inputCert);
certutilCert.run();
return nodePath;
}
private Path generateHttpCert(Path caPath, String... containerHostname) {
final Path httpPath = tempDir.resolve("test-http.p12");
TestableConsole inputHttp = TestableConsole.empty().silent()
.register("Do you want to use your own certificate authority? Respond with y/n?", "n")
.register("Enter CA password", "password")
.register("Enter certificate validity in days", "90")
.register("Enter alternative names (addresses) of this node [comma separated]", String.join(",", containerHostname))
.register("Enter HTTP certificate password", "password");
CertutilHttp certutilCert = new CertutilHttp(
caPath.toAbsolutePath().toString(),
httpPath.toAbsolutePath().toString(),
inputHttp);
certutilCert.run();
return httpPath;
}
}

View File

@@ -0,0 +1,98 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.integration;
import org.apache.commons.lang3.RandomStringUtils;
import org.graylog.datanode.configuration.variants.KeystoreInformation;
import org.graylog.security.certutil.CertutilCa;
import org.graylog.security.certutil.CertutilCert;
import org.graylog.security.certutil.CertutilHttp;
import org.graylog.security.certutil.console.TestableConsole;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.Enumeration;
public class DatanodeSecurityTestUtils {
public static KeyStore buildTruststore(KeystoreInformation ca) throws IOException, GeneralSecurityException {
try (FileInputStream fis = new FileInputStream(ca.location().toFile())) {
KeyStore caKeystore = KeyStore.getInstance("PKCS12");
caKeystore.load(fis, ca.password());
KeyStore trustStore = KeyStore.getInstance("PKCS12");
trustStore.load(null, null);
final Enumeration<String> aliases = caKeystore.aliases();
while (aliases.hasMoreElements()) {
final String alias = aliases.nextElement();
final Certificate cert = caKeystore.getCertificate(alias);
if (cert instanceof final X509Certificate x509Certificate) {
trustStore.setCertificateEntry(alias, x509Certificate);
}
}
return trustStore;
}
}
public static KeystoreInformation generateCa(Path dir) {
final Path certPath = dir.resolve("test-ca.p12");
final String password = RandomStringUtils.randomAlphabetic(10);
final TestableConsole input = TestableConsole.empty().silent()
.register("Enter CA password", password);
final CertutilCa command = new CertutilCa(certPath.toAbsolutePath().toString(), input);
command.run();
return new KeystoreInformation(certPath, password);
}
public static KeystoreInformation generateTransportCert(Path dir, KeystoreInformation ca, String... containerHostnames) {
final Path transportPath = dir.resolve("transport-" + RandomStringUtils.randomAlphabetic(10) + ".p12");
final String password = RandomStringUtils.randomAlphabetic(10);
TestableConsole inputCert = TestableConsole.empty().silent()
.register("Enter CA password", ca.passwordAsString())
.register("Enter datanode certificate password", password)
.register("Enter alternative names (addresses) of this node [comma separated]", String.join(",", containerHostnames));
CertutilCert certutilCert = new CertutilCert(
ca.location().toAbsolutePath().toString(),
transportPath.toAbsolutePath().toString(),
inputCert);
certutilCert.run();
return new KeystoreInformation(transportPath, password);
}
public static KeystoreInformation generateHttpCert(Path dir, KeystoreInformation ca, String... containerHostnames) {
final Path httpPath = dir.resolve("http-" + RandomStringUtils.randomAlphabetic(10) + ".p12");
final String password = RandomStringUtils.randomAlphabetic(10);
TestableConsole inputHttp = TestableConsole.empty().silent()
.register("Do you want to use your own certificate authority? Respond with y/n?", "n")
.register("Enter CA password", ca.passwordAsString())
.register("Enter certificate validity in days", "90")
.register("Enter alternative names (addresses) of this node [comma separated]", String.join(",", containerHostnames))
.register("Enter HTTP certificate password", password);
CertutilHttp certutilCert = new CertutilHttp(
ca.location().toAbsolutePath().toString(),
httpPath.toAbsolutePath().toString(),
inputHttp);
certutilCert.run();
return new KeystoreInformation(httpPath, password);
}
}