Datanode keystore implementation (#19536)

* Datanode keystore implementation

* Bind keystore check, remove legacy private key storage

* added license

* code cleanup, better cert signed verification

* added changelog

* code cleanup

data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java

@@ -19,11 +19,14 @@ package org.graylog.datanode.bindings;
 import com.google.inject.AbstractModule;
 import com.google.inject.multibindings.MapBinder;
 import org.graylog.datanode.bootstrap.preflight.DatanodeDirectoriesLockfileCheck;
+import org.graylog.datanode.bootstrap.preflight.DatanodeKeystoreCheck;
 import org.graylog.datanode.bootstrap.preflight.OpenSearchPreconditionsCheck;
 import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck;
 import org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync;
 import org.graylog.datanode.bootstrap.preflight.OpensearchDataDirCompatibilityCheck;
+import org.graylog2.bindings.providers.MongoConnectionProvider;
 import org.graylog2.bootstrap.preflight.PreflightCheck;
+import org.graylog2.database.MongoConnection;
 
 public class PreflightChecksBindings extends AbstractModule {
 
@@ -35,6 +38,11 @@ public class PreflightChecksBindings extends AbstractModule {
         addPreflightCheck(DatanodeDirectoriesLockfileCheck.class);
         addPreflightCheck(OpenSearchPreconditionsCheck.class);
         addPreflightCheck(OpensearchDataDirCompatibilityCheck.class);
+
+        // Mongodb is needed for legacy datanode storage, where we want to extract the certificate chain from
+        // mongodb and store it in local keystore
+        bind(MongoConnection.class).toProvider(MongoConnectionProvider.class);
+        addPreflightCheck(DatanodeKeystoreCheck.class);
     }
 
 

data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/DataNodeConfigurationPeriodical.java

@@ -18,22 +18,16 @@ package org.graylog.datanode.bootstrap.preflight;
 
 import com.google.common.collect.ImmutableList;
 import jakarta.inject.Inject;
-import jakarta.inject.Named;
 import jakarta.inject.Singleton;
 import org.bouncycastle.operator.OperatorException;
-import org.graylog.datanode.configuration.DatanodeConfiguration;
-import org.graylog.security.certutil.CertConstants;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.graylog.datanode.configuration.DatanodeKeystore;
+import org.graylog.datanode.configuration.DatanodeKeystoreException;
 import org.graylog.security.certutil.cert.CertificateChain;
 import org.graylog.security.certutil.cert.storage.CertChainMongoStorage;
 import org.graylog.security.certutil.cert.storage.CertChainStorage;
-import org.graylog.security.certutil.csr.CertificateAndPrivateKeyMerger;
-import org.graylog.security.certutil.csr.CsrGenerator;
 import org.graylog.security.certutil.csr.exceptions.CSRGenerationException;
 import org.graylog.security.certutil.csr.storage.CsrMongoStorage;
-import org.graylog.security.certutil.keystore.storage.KeystoreMongoStorage;
-import org.graylog.security.certutil.keystore.storage.location.KeystoreMongoCollections;
-import org.graylog.security.certutil.keystore.storage.location.KeystoreMongoLocation;
-import org.graylog.security.certutil.privatekey.PrivateKeyEncryptedFileStorage;
 import org.graylog2.cluster.NodeNotFoundException;
 import org.graylog2.cluster.nodes.DataNodeDto;
 import org.graylog2.cluster.nodes.NodeService;
@@ -47,8 +41,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.nio.file.Path;
-import java.security.KeyStore;
 import java.util.Collections;
 import java.util.Objects;
 import java.util.Optional;
@@ -62,36 +54,24 @@ public class DataNodeConfigurationPeriodical extends Periodical {
     private final DataNodeProvisioningService dataNodeProvisioningService;
     private final NodeService<DataNodeDto> nodeService;
     private final NodeId nodeId;
-    private final PrivateKeyEncryptedFileStorage privateKeyEncryptedStorage;
     private final CsrMongoStorage csrStorage;
-    private final CsrGenerator csrGenerator;
     private final CertChainStorage certMongoStorage;
-    private final CertificateAndPrivateKeyMerger certificateAndPrivateKeyMerger;
-    private final char[] passwordSecret;
 
-    private final KeystoreMongoStorage mongoKeyStorage;
+    private final DatanodeKeystore datanodeKeystore;
 
     @Inject
     public DataNodeConfigurationPeriodical(final DataNodeProvisioningService dataNodeProvisioningService,
                                            final NodeService<DataNodeDto> nodeService,
                                            final NodeId nodeId,
                                            final CsrMongoStorage csrStorage,
-                                           final CsrGenerator csrGenerator,
                                            final CertChainMongoStorage certMongoStorage,
-                                           final CertificateAndPrivateKeyMerger certificateAndPrivateKeyMerger,
-                                           final @Named("password_secret") String passwordSecret,
-                                           final DatanodeConfiguration datanodeConfiguration, KeystoreMongoStorage mongoKeyStorage) throws IOException {
+                                           DatanodeKeystore datanodeKeystore) {
         this.dataNodeProvisioningService = dataNodeProvisioningService;
         this.nodeService = nodeService;
         this.nodeId = nodeId;
         this.csrStorage = csrStorage;
-        this.csrGenerator = csrGenerator;
         this.certMongoStorage = certMongoStorage;
-        this.certificateAndPrivateKeyMerger = certificateAndPrivateKeyMerger;
-        // TODO: merge with real storage
-        this.privateKeyEncryptedStorage = new PrivateKeyEncryptedFileStorage(datanodeConfiguration.datanodeDirectories().createConfigurationFile(Path.of("privateKey.cert")));
-        this.passwordSecret = passwordSecret.toCharArray();
-        this.mongoKeyStorage = mongoKeyStorage;
+        this.datanodeKeystore = datanodeKeystore;
     }
 
     @Override
@@ -111,7 +91,8 @@ public class DataNodeConfigurationPeriodical extends Periodical {
             switch (state) {
                 case CONFIGURED -> writeCsr(c);
                 case SIGNED -> readSignedCertificate(c);
-                case STARTUP_TRIGGER ->  dataNodeProvisioningService.changeState(nodeId.getNodeId(), DataNodeProvisioningConfig.State.STARTUP_REQUESTED);
+                case STARTUP_TRIGGER ->
+                        dataNodeProvisioningService.changeState(nodeId.getNodeId(), DataNodeProvisioningConfig.State.STARTUP_REQUESTED);
             }
         });
     }
@@ -121,28 +102,23 @@ public class DataNodeConfigurationPeriodical extends Periodical {
             LOG.error("Config entry in signed state, but no certificate data present in Mongo");
         } else {
             try {
-                final Optional<CertificateChain> certificateChain = certMongoStorage.readCertChain(nodeId.getNodeId());
-                if (certificateChain.isPresent()) {
-                    final char[] secret = passwordSecret;
-                    KeyStore nodeKeystore = certificateAndPrivateKeyMerger.merge(
-                            certificateChain.get(),
-                            privateKeyEncryptedStorage,
-                            secret,
-                            secret,
-                            CertConstants.DATANODE_KEY_ALIAS
-                    );
-
-                    mongoKeyStorage.writeKeyStore(KeystoreMongoLocation.datanode(nodeId), nodeKeystore, secret, secret);
-
-                    //should be in one transaction, but we miss transactions...
-                    dataNodeProvisioningService.changeState(nodeId.getNodeId(), DataNodeProvisioningConfig.State.STORED);
-                }
+                certMongoStorage.readCertChain(nodeId.getNodeId())
+                        .ifPresent(this::processCertificateChain);
             } catch (Exception ex) {
                 LOG.error("Config entry in signed state, but wrong certificate data present in Mongo");
             }
         }
     }
 
+    private void processCertificateChain(CertificateChain certificateChain) {
+        try {
+            datanodeKeystore.replaceCertificatesInKeystore(certificateChain);
+            dataNodeProvisioningService.changeState(nodeId.getNodeId(), DataNodeProvisioningConfig.State.STORED);
+        } catch (DatanodeKeystoreException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private void writeCsr(DataNodeProvisioningConfig cfg) {
         try {
             final var node = nodeService.byNodeId(nodeId);
@@ -150,10 +126,12 @@ public class DataNodeConfigurationPeriodical extends Periodical {
                     .addAll(Optional.ofNullable(cfg.altNames()).orElse(Collections.emptyList()))
                     .addAll(determineAltNames())
                     .build();
-            final var csr = csrGenerator.generateCSR(passwordSecret, node.getHostname(), altNames, privateKeyEncryptedStorage);
+            final PKCS10CertificationRequest csr = datanodeKeystore.createCertificateSigningRequest(node.getHostname(), altNames);
             csrStorage.writeCsr(csr, nodeId.getNodeId());
             LOG.info("created CSR for this node");
-        } catch (CSRGenerationException | IOException | NodeNotFoundException | OperatorException ex) {
+        } catch (CSRGenerationException | IOException | NodeNotFoundException | OperatorException |
+                 DatanodeKeystoreException
+                ex) {
             LOG.error("error generating a CSR: " + ex.getMessage(), ex);
             dataNodeProvisioningService.save(cfg.asError(ex.getMessage()));
         }

data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/DatanodeKeystoreCheck.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.bootstrap.preflight;
+
+import jakarta.inject.Inject;
+import org.graylog.datanode.configuration.DatanodeKeystore;
+import org.graylog.security.certutil.CertRequest;
+import org.graylog.security.certutil.CertificateGenerator;
+import org.graylog.security.certutil.KeyPair;
+import org.graylog2.bootstrap.preflight.PreflightCheck;
+import org.graylog2.bootstrap.preflight.PreflightCheckException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyStore;
+import java.security.Security;
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * This check verifies that each datanode has a private key configured. It may be needed right now, but at the moment
+ * we'll start provisioning and create certificate signing requests, we may be sure that there is one private key
+ * available.
+ * ·<br>
+ * Additionally, this check is able to restore existing keystore persisted in mongodb. It may hold valid certificate
+ * that we want to reuse. Otherwise, the node would undergo signing again.
+ */
+public class DatanodeKeystoreCheck implements PreflightCheck {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DatanodeKeystoreCheck.class);
+    public static final Duration DEFAULT_SELFSIGNED_CERT_VALIDITY = Duration.ofDays(99 * 365);
+
+    private final DatanodeKeystore datanodeKeystore;
+    private final LegacyDatanodeKeystoreProvider legacyDatanodeKeystoreProvider;
+
+    @Inject
+    public DatanodeKeystoreCheck(DatanodeKeystore datanodeKeystore, LegacyDatanodeKeystoreProvider legacyDatanodeKeystoreProvider) {
+        this.datanodeKeystore = datanodeKeystore;
+        this.legacyDatanodeKeystoreProvider = legacyDatanodeKeystoreProvider;
+    }
+
+    @Override
+    public void runCheck() throws PreflightCheckException {
+        if (!datanodeKeystore.exists()) {
+            LOG.info("Creating keystore for this data node");
+            try {
+                final Optional<KeyStore> legacyKeystore = legacyDatanodeKeystoreProvider.get();
+                if (legacyKeystore.isPresent()) { // remove this branch latest with 7.0 release
+                    LOG.info("Legacy keystore discovered, converting to local file");
+                    datanodeKeystore.create(legacyKeystore.get());
+                    legacyDatanodeKeystoreProvider.deleteLocalPrivateKey();
+                } else {
+                    datanodeKeystore.create(generateKeyPair());
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            LOG.debug("Private key for this data node already exists, skipping creation.");
+        }
+    }
+
+    private static KeyPair generateKeyPair() throws Exception {
+        final CertRequest certRequest = CertRequest.selfSigned(DatanodeKeystore.DATANODE_KEY_ALIAS)
+                .isCA(false)
+                .validity(DEFAULT_SELFSIGNED_CERT_VALIDITY);
+
+        return CertificateGenerator.generate(certRequest);
+    }
+}

data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/LegacyDatanodeKeystoreProvider.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.bootstrap.preflight;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import org.graylog.datanode.configuration.DatanodeConfiguration;
+import org.graylog.datanode.configuration.DatanodeDirectories;
+import org.graylog.datanode.configuration.DatanodeKeystore;
+import org.graylog.datanode.configuration.DatanodeKeystoreException;
+import org.graylog.security.certutil.CertConstants;
+import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.graylog.security.certutil.keystore.storage.location.KeystoreMongoLocation;
+import org.graylog2.cluster.certificates.CertificatesService;
+import org.graylog2.plugin.system.NodeId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.util.Base64;
+import java.util.Optional;
+
+import static org.graylog.security.certutil.CertConstants.PKCS12;
+
+/**
+ * Remove in 7.0 release. All legacy mongodb stored keystores for datanodes should be migrated to local files
+ * or recreated from scratch.
+ */
+@Deprecated(forRemoval = true)
+public class LegacyDatanodeKeystoreProvider {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LegacyDatanodeKeystoreProvider.class);
+
+    private final CertificatesService certificatesService;
+    private final NodeId nodeId;
+    private final String passwordSecret;
+
+    private final DatanodeDirectories datanodeDirectories;
+
+    @Inject
+    public LegacyDatanodeKeystoreProvider(CertificatesService certificatesService, NodeId nodeId, final @Named("password_secret") String passwordSecret, DatanodeConfiguration datanodeConfiguration) {
+        this.certificatesService = certificatesService;
+        this.nodeId = nodeId;
+        this.passwordSecret = passwordSecret;
+        this.datanodeDirectories = datanodeConfiguration.datanodeDirectories();
+    }
+
+    public Optional<KeyStore> get() throws KeyStoreStorageException {
+        return loadKeystore().filter(this::isValidKeyAndCert);
+    }
+
+    private boolean isValidKeyAndCert(KeyStore keystore) {
+        try {
+            return hasPrivateKey(keystore) && DatanodeKeystore.isSignedCertificateChain(keystore);
+        } catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException |
+                 DatanodeKeystoreException e) {
+            LOG.warn("Failed to obtain legacy keystore, ignoring it", e);
+            return false;
+        }
+    }
+
+    private boolean hasPrivateKey(KeyStore keystore) throws KeyStoreException, NoSuchAlgorithmException, UnrecoverableKeyException {
+        return keystore.getKey(CertConstants.DATANODE_KEY_ALIAS, passwordSecret.toCharArray()) != null;
+    }
+
+    private Optional<KeyStore> loadKeystore() throws KeyStoreStorageException {
+        final Optional<String> keystoreAsString = readEncodedCertFromDatabase();
+        if (keystoreAsString.isPresent()) {
+            try (ByteArrayInputStream bais = new ByteArrayInputStream(Base64.getDecoder().decode(keystoreAsString.get()))) {
+                KeyStore keyStore = KeyStore.getInstance(PKCS12);
+                keyStore.load(bais, passwordSecret.toCharArray());
+                return Optional.of(keyStore);
+            } catch (Exception ex) {
+                throw new KeyStoreStorageException("Failed to load keystore from Mongo collection for node " + nodeId.getNodeId(), ex);
+            }
+        }
+        return Optional.empty();
+    }
+
+    private Optional<String> readEncodedCertFromDatabase() {
+        KeystoreMongoLocation location = KeystoreMongoLocation.datanode(nodeId);
+        return certificatesService.readCert(location);
+    }
+
+    public void deleteLocalPrivateKey() {
+        final Path localPrivateKey = datanodeDirectories.getConfigurationTargetDir().resolve("privateKey.cert");
+        if (Files.exists(localPrivateKey)) {
+            try {
+                Files.delete(localPrivateKey);
+            } catch (IOException e) {
+                LOG.warn("Failed to delete legacy datanode private key", e);
+            }
+        }
+    }
+}

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystore.java

@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.configuration;
+
+import com.google.common.eventbus.EventBus;
+import jakarta.annotation.Nonnull;
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.graylog.security.certutil.CertConstants;
+import org.graylog.security.certutil.KeyPair;
+import org.graylog.security.certutil.cert.CertificateChain;
+import org.graylog.security.certutil.csr.CsrGenerator;
+import org.graylog.security.certutil.csr.InMemoryKeystoreInformation;
+import org.graylog.security.certutil.csr.exceptions.CSRGenerationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.InvalidKeyException;
+import java.security.Key;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.Security;
+import java.security.SignatureException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Date;
+import java.util.List;
+
+import static org.graylog.security.certutil.CertConstants.PKCS12;
+
+public class DatanodeKeystore {
+
+    static {
+        Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());
+    }
+
+
+    private static final Logger LOG = LoggerFactory.getLogger(DatanodeKeystore.class);
+    private final DatanodeDirectories datanodeDirectories;
+    private final String passwordSecret;
+
+    public static final Path DATANODE_KEYSTORE_FILE = Path.of("keystore.jks");
+    public static String DATANODE_KEY_ALIAS = CertConstants.DATANODE_KEY_ALIAS;
+    private final EventBus eventBus;
+
+    @Inject
+    public DatanodeKeystore(DatanodeConfiguration configuration, final @Named("password_secret") String passwordSecret, EventBus eventBus) {
+        this(configuration.datanodeDirectories(), passwordSecret, eventBus);
+    }
+
+    DatanodeKeystore(DatanodeDirectories datanodeDirectories, String passwordSecret, EventBus eventBus) {
+        this.datanodeDirectories = datanodeDirectories;
+        this.passwordSecret = passwordSecret;
+        this.eventBus = eventBus;
+    }
+
+    public synchronized boolean exists() {
+        return Files.exists(keystorePath());
+    }
+
+    public synchronized boolean hasSignedCertificate() throws DatanodeKeystoreException {
+        return isSignedCertificateChain(loadKeystore());
+    }
+
+    public synchronized static boolean isSignedCertificateChain(KeyStore keystore) throws DatanodeKeystoreException {
+        try {
+
+            final Certificate[] certificateChain = keystore.getCertificateChain(DATANODE_KEY_ALIAS);
+
+            if (certificateChain.length < 2) {
+                // only one cert, it's a self-signed cert!
+                return false;
+            }
+            try {
+                // let's take first cert, which is the datanode. It should be signed by a private key that belongs
+                // to the public key in the second certificate
+                certificateChain[0].verify(certificateChain[1].getPublicKey());
+                return true;
+            } catch (CertificateException | NoSuchAlgorithmException | InvalidKeyException |
+                     NoSuchProviderException | SignatureException e) {
+                return false;
+            }
+
+        } catch (KeyStoreException e) {
+            throw new DatanodeKeystoreException(e);
+        }
+    }
+
+    @Nonnull
+    private Path keystorePath() {
+        return datanodeDirectories.getConfigurationTargetDir().resolve(DATANODE_KEYSTORE_FILE);
+    }
+
+    public synchronized KeyStore create(KeyPair keyPair) throws DatanodeKeystoreException {
+        try {
+            return persistKeystore(keyPair.toKeystore(DATANODE_KEY_ALIAS, passwordSecret.toCharArray()));
+        } catch (Exception e) {
+            throw new DatanodeKeystoreException(e);
+        }
+    }
+
+    public synchronized KeyStore create(KeyStore keystore) throws DatanodeKeystoreException {
+        return persistKeystore(keystore);
+    }
+
+    public synchronized void replaceCertificatesInKeystore(CertificateChain certificateChain) throws DatanodeKeystoreException {
+        try {
+            final KeyStore keystore = loadKeystore();
+            Key privateKey = keystore.getKey(DATANODE_KEY_ALIAS, passwordSecret.toCharArray());
+            // replace the existing self-signed certificates chain with the signed chain from the event
+            keystore.setKeyEntry(DATANODE_KEY_ALIAS, privateKey, passwordSecret.toCharArray(), certificateChain.toCertificateChainArray());
+            LOG.info("Persisting signed certificates to the datanode keystore finished");
+            persistKeystore(keystore);
+        } catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException e) {
+            throw new DatanodeKeystoreException(e);
+        }
+    }
+
+    private KeyStore persistKeystore(KeyStore keystore) throws DatanodeKeystoreException {
+        try (FileOutputStream fos = new FileOutputStream(keystorePath().toFile())) {
+            keystore.store(fos, passwordSecret.toCharArray());
+        } catch (IOException | CertificateException | KeyStoreException | NoSuchAlgorithmException e) {
+            throw new DatanodeKeystoreException(e);
+        }
+        triggerChangeEvent(keystore);
+        return keystore;
+    }
+
+    private void triggerChangeEvent(KeyStore keystore) throws DatanodeKeystoreException {
+        if (isSignedCertificateChain(keystore)) {
+            eventBus.post(new DatanodeKeystoreChangedEvent());
+        }
+    }
+
+    public synchronized KeyStore loadKeystore() throws DatanodeKeystoreException {
+        try (FileInputStream fis = new FileInputStream(keystorePath().toFile())) {
+            KeyStore keystore = KeyStore.getInstance(PKCS12);
+            keystore.load(fis, passwordSecret.toCharArray());
+            return keystore;
+        } catch (IOException | CertificateException | KeyStoreException | NoSuchAlgorithmException e) {
+            throw new DatanodeKeystoreException(e);
+        }
+    }
+
+    public synchronized PKCS10CertificationRequest createCertificateSigningRequest(String hostname, List<String> altNames) throws DatanodeKeystoreException, CSRGenerationException {
+        final InMemoryKeystoreInformation keystore = new InMemoryKeystoreInformation(loadKeystore(), passwordSecret.toCharArray());
+        return CsrGenerator.generateCSR(keystore, DATANODE_KEY_ALIAS, hostname, altNames);
+    }
+
+    public Date getCertificateExpiration() {
+
+        try {
+            final X509Certificate datanodeCert = (X509Certificate) loadKeystore().getCertificate(DATANODE_KEY_ALIAS);
+            return datanodeCert.getNotAfter();
+        } catch (KeyStoreException | DatanodeKeystoreException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystoreChangedEvent.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.configuration;
+
+/**
+ * This event will be triggered every time a datanode private key or certificate changes. Other parts of the system,
+ * that use the keystore (opensearch process, jersey) should react to this event and refresh their security setups
+ */
+public class DatanodeKeystoreChangedEvent {
+}

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeKeystoreException.java

@@ -14,16 +14,10 @@
  * along with this program. If not, see
  * <http://www.mongodb.com/licensing/server-side-public-license>.
  */
-package org.graylog.datanode.configuration.variants;
+package org.graylog.datanode.configuration;
 
-import java.nio.file.Path;
-
-public record KeystoreInformation(Path location, char[] password) {
-    public KeystoreInformation(Path location, String password) {
-        this(location, password.toCharArray());
-    }
-
-    public String passwordAsString() {
-        return new String(password());
+public class DatanodeKeystoreException extends Exception {
+    public DatanodeKeystoreException(Throwable cause) {
+        super(cause);
     }
 }

data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationProvider.java

@@ -23,7 +23,7 @@ import jakarta.inject.Provider;
 import jakarta.inject.Singleton;
 import org.graylog.datanode.Configuration;
 import org.graylog.datanode.configuration.variants.InSecureConfiguration;
-import org.graylog.datanode.configuration.variants.MongoCertSecureConfiguration;
+import org.graylog.datanode.configuration.variants.LocalKeystoreSecureConfiguration;
 import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration;
 import org.graylog.datanode.configuration.variants.SecurityConfigurationVariant;
 import org.graylog.datanode.configuration.variants.UploadedCertFilesSecureConfiguration;
@@ -46,7 +46,7 @@ import java.util.stream.Collectors;
 public class OpensearchConfigurationProvider implements Provider<OpensearchConfiguration> {
     private final Configuration localConfiguration;
     private final UploadedCertFilesSecureConfiguration uploadedCertFilesSecureConfiguration;
-    private final MongoCertSecureConfiguration mongoCertSecureConfiguration;
+    private final LocalKeystoreSecureConfiguration mongoCertSecureConfiguration;
     private final InSecureConfiguration inSecureConfiguration;
     private final DatanodeConfiguration datanodeConfiguration;
     private final byte[] signingKey;
@@ -62,7 +62,7 @@ public class OpensearchConfigurationProvider implements Provider<OpensearchConfi
     public OpensearchConfigurationProvider(final Configuration localConfiguration,
                                            final DatanodeConfiguration datanodeConfiguration,
                                            final UploadedCertFilesSecureConfiguration uploadedCertFilesSecureConfiguration,
-                                           final MongoCertSecureConfiguration mongoCertSecureConfiguration,
+                                           final LocalKeystoreSecureConfiguration localKeystoreSecureConfiguration,
                                            final InSecureConfiguration inSecureConfiguration,
                                            final NodeService<DataNodeDto> nodeService,
                                            final @Named("password_secret") String passwordSecret,
@@ -70,7 +70,7 @@ public class OpensearchConfigurationProvider implements Provider<OpensearchConfi
         this.localConfiguration = localConfiguration;
         this.datanodeConfiguration = datanodeConfiguration;
         this.uploadedCertFilesSecureConfiguration = uploadedCertFilesSecureConfiguration;
-        this.mongoCertSecureConfiguration = mongoCertSecureConfiguration;
+        this.mongoCertSecureConfiguration = localKeystoreSecureConfiguration;
         this.inSecureConfiguration = inSecureConfiguration;
         this.signingKey = passwordSecret.getBytes(StandardCharsets.UTF_8);
         this.nodeService = nodeService;

data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java

@@ -16,8 +16,8 @@
  */
 package org.graylog.datanode.configuration;
 
-import org.graylog.datanode.configuration.variants.KeystoreInformation;
 import org.graylog.security.certutil.CertConstants;
+import org.graylog.security.certutil.csr.FilesystemKeystoreInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,14 +44,14 @@ public class TruststoreCreator {
         return new TruststoreCreator();
     }
 
-    public TruststoreCreator addRootCert(final String name, KeystoreInformation keystoreInformation,
+    public TruststoreCreator addRootCert(final String name, FilesystemKeystoreInformation keystoreInformation,
                                          final String alias) throws IOException, GeneralSecurityException {
         final X509Certificate rootCert = findRootCert(keystoreInformation.location(), keystoreInformation.password(), alias);
         rootCertificates.put(name, rootCert);
         return this;
     }
 
-    public KeystoreInformation persist(final Path truststorePath, final char[] truststorePassword) throws IOException, GeneralSecurityException {
+    public FilesystemKeystoreInformation persist(final Path truststorePath, final char[] truststorePassword) throws IOException, GeneralSecurityException {
         KeyStore trustStore = KeyStore.getInstance(CertConstants.PKCS12);
         trustStore.load(null, null);
 
@@ -63,7 +63,7 @@ public class TruststoreCreator {
         try (final FileOutputStream fileOutputStream = new FileOutputStream(truststorePath.toFile())) {
             trustStore.store(fileOutputStream, truststorePassword);
         }
-        return new KeystoreInformation(truststorePath, truststorePassword);
+        return new FilesystemKeystoreInformation(truststorePath, truststorePassword);
     }
 
 

data-node/src/main/java/org/graylog/datanode/configuration/variants/LocalKeystoreSecureConfiguration.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.configuration.variants;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import org.graylog.datanode.Configuration;
+import org.graylog.datanode.configuration.DatanodeConfiguration;
+import org.graylog.datanode.configuration.DatanodeKeystore;
+import org.graylog.datanode.configuration.DatanodeKeystoreException;
+import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.graylog.security.certutil.csr.FilesystemKeystoreInformation;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+public final class LocalKeystoreSecureConfiguration extends SecureConfiguration {
+    private final DatanodeKeystore datanodeKeystore;
+    private final char[] secret;
+
+    @Inject
+    public LocalKeystoreSecureConfiguration(final DatanodeKeystore datanodeKeystore,
+                                            final DatanodeConfiguration datanodeConfiguration,
+                                            final @Named("password_secret") String passwordSecret
+    ) {
+        super(datanodeConfiguration);
+        this.datanodeKeystore = datanodeKeystore;
+        this.secret = passwordSecret.toCharArray();
+    }
+
+    @Override
+    public boolean isConfigured(Configuration localConfiguration) {
+        try {
+            return datanodeKeystore.exists() && datanodeKeystore.hasSignedCertificate();
+        } catch (DatanodeKeystoreException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public OpensearchSecurityConfiguration build() throws KeyStoreStorageException, IOException, GeneralSecurityException {
+
+        final Path targetTransportKeystoreLocation = getTransportKeystoreLocation();
+        final Path targetHttpKeystoreLocation = getHttpKeystoreLocation();
+
+        try {
+            final KeyStore datanodeKeystore = this.datanodeKeystore.loadKeystore();
+            copy(datanodeKeystore, targetTransportKeystoreLocation, secret);
+            copy(datanodeKeystore, targetHttpKeystoreLocation, secret);
+
+            return new OpensearchSecurityConfiguration(
+                    new FilesystemKeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), secret),
+                    new FilesystemKeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), secret)
+            );
+        } catch (DatanodeKeystoreException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void copy(final KeyStore originalKeystore, final Path targetLocation, final char[] password) {
+        try (FileOutputStream fos = new FileOutputStream(targetLocation.toFile())) {
+            originalKeystore.store(fos, password);
+        } catch (IOException | CertificateException | KeyStoreException | NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

data-node/src/main/java/org/graylog/datanode/configuration/variants/MongoCertSecureConfiguration.java

@@ -1,109 +0,0 @@
-/*
- * Copyright (C) 2020 Graylog, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- */
-package org.graylog.datanode.configuration.variants;
-
-import jakarta.inject.Inject;
-import jakarta.inject.Named;
-import jakarta.validation.constraints.NotNull;
-import org.graylog.datanode.Configuration;
-import org.graylog.datanode.configuration.DatanodeConfiguration;
-import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
-import org.graylog.security.certutil.keystore.storage.KeystoreFileStorage;
-import org.graylog.security.certutil.keystore.storage.KeystoreMongoStorage;
-import org.graylog.security.certutil.keystore.storage.location.KeystoreMongoLocation;
-import org.graylog2.cluster.certificates.CertificatesService;
-import org.graylog2.plugin.system.NodeId;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.security.GeneralSecurityException;
-import java.security.KeyStore;
-import java.util.Optional;
-
-public final class MongoCertSecureConfiguration extends SecureConfiguration {
-
-    private final NodeId nodeId;
-    private final CertificatesService certificatesService;
-
-    private final char[] secret;
-
-    private final char[] mongoKeystorePassword;
-
-    private final KeystoreMongoStorage mongoKeyStorage;
-    private final KeystoreFileStorage fileKeyStorage;
-
-    @Inject
-    public MongoCertSecureConfiguration(final DatanodeConfiguration datanodeConfiguration,
-                                        final NodeId nodeId,
-                                        final @Named("password_secret") String passwordSecret,
-                                        final CertificatesService certificatesService, KeystoreMongoStorage mongoKeyStore, KeystoreFileStorage fileKeyStore
-    ) {
-        super(datanodeConfiguration);
-        this.nodeId = nodeId;
-        this.certificatesService = certificatesService;
-
-        this.secret = passwordSecret.toCharArray();
-        this.mongoKeyStorage = mongoKeyStore;
-        this.fileKeyStorage = fileKeyStore;
-
-        //TODO: matches line 123 of DataNodePreflightGeneratePeriodical, but both need to be changed
-        this.mongoKeystorePassword = secret;
-    }
-
-    @Override
-    public boolean isConfigured(Configuration localConfiguration) {
-        return certificatesService.hasCert(getMongoKeystoreLocation());
-    }
-
-    @Override
-    public OpensearchSecurityConfiguration build() throws KeyStoreStorageException, IOException, GeneralSecurityException {
-
-        final Path targetTransportKeystoreLocation = getTransportKeystoreLocation();
-        final Path targetHttpKeystoreLocation = getHttpKeystoreLocation();
-
-
-        // this will take the mongodb-stored keys and persist them on a disk, in the opensearch configuration directory
-        final KeystoreMongoLocation mongoKeystoreLocation = getMongoKeystoreLocation();
-        reEncypt(mongoKeystoreLocation, mongoKeystorePassword, targetTransportKeystoreLocation, secret);
-        reEncypt(mongoKeystoreLocation, mongoKeystorePassword, targetHttpKeystoreLocation, secret);
-
-        return new OpensearchSecurityConfiguration(
-                new KeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), secret),
-                new KeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), secret)
-        );
-    }
-
-    @NotNull
-    private KeystoreMongoLocation getMongoKeystoreLocation() {
-        return KeystoreMongoLocation.datanode(nodeId);
-    }
-
-    private void reEncypt(final KeystoreMongoLocation originalLocation,
-                          final char[] originalPassword,
-                          final Path targetLocation,
-                          final char[] newPassword
-    ) throws KeyStoreStorageException {
-
-        final Optional<KeyStore> keyStore = mongoKeyStorage.readKeyStore(originalLocation, originalPassword);
-        if (keyStore.isPresent()) {
-            final KeyStore originalKeystore = keyStore.get();
-            fileKeyStorage.writeKeyStore(targetLocation, originalKeystore, originalPassword, newPassword);
-        } else {
-            throw new KeyStoreStorageException("No keystore present in : " + originalLocation);
-        }
-    }
-}

data-node/src/main/java/org/graylog/datanode/configuration/variants/OpensearchSecurityConfiguration.java

@@ -24,6 +24,7 @@ import org.apache.commons.lang3.RandomStringUtils;
 import org.graylog.datanode.configuration.DatanodeConfiguration;
 import org.graylog.datanode.configuration.TruststoreCreator;
 import org.graylog.security.certutil.CertConstants;
+import org.graylog.security.certutil.csr.FilesystemKeystoreInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,12 +54,12 @@ public class OpensearchSecurityConfiguration {
     private static final String TRUSTSTORE_FORMAT = "PKCS12";
     private static final Path TRUSTSTORE_FILE = Path.of("datanode-truststore.p12");
 
-    private final KeystoreInformation transportCertificate;
-    private final KeystoreInformation httpCertificate;
-    private KeystoreInformation truststore;
+    private final FilesystemKeystoreInformation transportCertificate;
+    private final FilesystemKeystoreInformation httpCertificate;
+    private FilesystemKeystoreInformation truststore;
     private String opensearchHeap;
 
-    public OpensearchSecurityConfiguration(KeystoreInformation transportCertificate, KeystoreInformation httpCertificate) {
+    public OpensearchSecurityConfiguration(FilesystemKeystoreInformation transportCertificate, FilesystemKeystoreInformation httpCertificate) {
         this.transportCertificate = transportCertificate;
         this.httpCertificate = httpCertificate;
     }
@@ -156,15 +157,15 @@ public class OpensearchSecurityConfiguration {
         return !Objects.isNull(httpCertificate) && !Objects.isNull(transportCertificate);
     }
 
-    public KeystoreInformation getTransportCertificate() {
+    public FilesystemKeystoreInformation getTransportCertificate() {
         return transportCertificate;
     }
 
-    public KeystoreInformation getHttpCertificate() {
+    public FilesystemKeystoreInformation getHttpCertificate() {
         return httpCertificate;
     }
 
-    public KeystoreInformation getTruststore() {
+    public FilesystemKeystoreInformation getTruststore() {
         return truststore;
     }
 
@@ -188,7 +189,7 @@ public class OpensearchSecurityConfiguration {
         return config.build();
     }
 
-    private void logCertificateInformation(String certificateType, KeystoreInformation keystore) throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException {
+    private void logCertificateInformation(String certificateType, FilesystemKeystoreInformation keystore) throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException {
         final KeyStore instance = KeyStore.getInstance(KEYSTORE_FORMAT);
         try (final FileInputStream is = new FileInputStream(keystore.location().toFile())) {
             instance.load(is, keystore.password());

data-node/src/main/java/org/graylog/datanode/configuration/variants/SecureConfiguration.java

@@ -23,7 +23,7 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.function.Supplier;
 
-public sealed abstract class SecureConfiguration implements SecurityConfigurationVariant permits MongoCertSecureConfiguration, UploadedCertFilesSecureConfiguration {
+public abstract class SecureConfiguration implements SecurityConfigurationVariant {
 
     /**
      * This filename is used only internally - we copy user-provided certificates to this location and

data-node/src/main/java/org/graylog/datanode/configuration/variants/UploadedCertFilesSecureConfiguration.java

@@ -22,6 +22,7 @@ import org.graylog.datanode.Configuration;
 import org.graylog.datanode.configuration.DatanodeConfiguration;
 import org.graylog.datanode.configuration.OpensearchConfigurationException;
 import org.graylog.security.certutil.ca.exceptions.KeyStoreStorageException;
+import org.graylog.security.certutil.csr.FilesystemKeystoreInformation;
 import org.graylog.security.certutil.keystore.storage.KeystoreFileStorage;
 
 import java.io.IOException;
@@ -129,8 +130,8 @@ public final class UploadedCertFilesSecureConfiguration extends SecureConfigurat
                 targetHttpKeystoreLocation);
 
         return new OpensearchSecurityConfiguration(
-                new KeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), transportOTP),
-                new KeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), httpOTP)
+                new FilesystemKeystoreInformation(targetTransportKeystoreLocation.toAbsolutePath(), transportOTP),
+                new FilesystemKeystoreInformation(targetHttpKeystoreLocation.toAbsolutePath(), httpOTP)
         );
     }
 

data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java

@@ -44,12 +44,13 @@ import org.glassfish.jersey.server.ResourceConfig;
 import org.glassfish.jersey.server.ServerProperties;
 import org.glassfish.jersey.server.model.Resource;
 import org.graylog.datanode.Configuration;
-import org.graylog.datanode.configuration.variants.KeystoreInformation;
 import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration;
 import org.graylog.datanode.opensearch.OpensearchConfigurationChangeEvent;
 import org.graylog.datanode.opensearch.configuration.OpensearchConfiguration;
 import org.graylog.datanode.rest.config.SecuredNodeAnnotationFilter;
 import org.graylog.security.certutil.CertConstants;
+import org.graylog.security.certutil.csr.FilesystemKeystoreInformation;
+import org.graylog.security.certutil.csr.KeystoreInformation;
 import org.graylog2.configuration.TLSProtocolsConfiguration;
 import org.graylog2.plugin.inject.Graylog2Module;
 import org.graylog2.rest.MoreMediaTypes;
@@ -259,7 +260,7 @@ public class JerseyService extends AbstractIdleService {
         return httpServer;
     }
 
-    private SSLEngineConfigurator buildSslEngineConfigurator(KeystoreInformation keystoreInformation)
+    private SSLEngineConfigurator buildSslEngineConfigurator(FilesystemKeystoreInformation keystoreInformation)
             throws GeneralSecurityException, IOException {
         if (keystoreInformation == null || !Files.isRegularFile(keystoreInformation.location()) || !Files.isReadable(keystoreInformation.location())) {
             throw new IllegalArgumentException("Unreadable to read private key");
@@ -280,7 +281,7 @@ public class JerseyService extends AbstractIdleService {
         return sslEngineConfigurator;
     }
 
-    private static KeyStore readKeystore(KeystoreInformation keystoreInformation) {
+    private static KeyStore readKeystore(FilesystemKeystoreInformation keystoreInformation) {
         try (var in = Files.newInputStream(keystoreInformation.location())) {
             KeyStore caKeystore = KeyStore.getInstance(CertConstants.PKCS12);
             caKeystore.load(in, keystoreInformation.password());