opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix repeated opensearch startup during HEALTH_CHECK_FAILED (#19335)

Browse Source 
* Fix repeated opensearch startup during HEALTH_CHECK_FAILED

* Set network.publish_host to hostname for all datanodes

* Verify CA subject name in DatanodeProvisioningIT

* add `DataNodeStatus` and check that data node is `CONNECTED && AVAILABLE` to avoid race condition between ProvisioningPeriodical and NodeHeartbeat

* Simplify TruststoreCreator

---------

Co-authored-by: Matthias Oesterheld <matthias.oesterheld@graylog.com>
This commit is contained in:
Tomas Dvorak
2024-05-15 10:24:37 +02:00
committed by GitHub
parent e7965750eb
commit 269afe7ede
6 changed files with 87 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationProvider.java
View File

@@ -144,7 +144,7 @@ public class OpensearchConfigurationProvider implements Provider<OpensearchConfi
config.put("path.repo", localConfiguration.getPathRepo());
}
//config.put("network.publish_host", Tools.getLocalCanonicalHostname());
config.put("network.publish_host", localConfiguration.getHostname());
if (localConfiguration.getOpensearchDebug() != null && !localConfiguration.getOpensearchDebug().isBlank()) {
config.put("logger.org.opensearch", localConfiguration.getOpensearchDebug());

59
data-node/src/main/java/org/graylog/datanode/configuration/RootCertificateFinder.java
View File

@@ -1,59 +0,0 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.configuration;
import org.graylog.security.certutil.CertConstants;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.Arrays;
class RootCertificateFinder {
X509Certificate findRootCert(Path keystorePath,
char[] password,
final String alias) throws IOException, GeneralSecurityException {
final KeyStore keystore = loadKeystore(keystorePath, password);
final Certificate[] certs = keystore.getCertificateChain(alias);
return Arrays.stream(certs)
.filter(cert -> cert instanceof X509Certificate)
.map(cert -> (X509Certificate) cert)
.filter(cert -> isRootCaCertificate(cert) || certs.length == 1)//TODO: certs.length == 1 may be temporary, our merged does not create a proper cert chain, it seems
.findFirst()
.orElseThrow(() -> new KeyStoreException("Keystore does not contain root X509Certificate in the certificate chain!"));
}
private boolean isRootCaCertificate(X509Certificate cert) {
return cert.getSubjectX500Principal().equals(cert.getIssuerX500Principal());
}
private KeyStore loadKeystore(final Path keystorePath,
final char[] password) throws IOException, GeneralSecurityException {
KeyStore nodeKeystore = KeyStore.getInstance(CertConstants.PKCS12);
try (final FileInputStream is = new FileInputStream(keystorePath.toFile())) {
nodeKeystore.load(is, password);
}
return nodeKeystore;
}
}

44
data-node/src/main/java/org/graylog/datanode/configuration/TruststoreCreator.java
View File

@@ -21,12 +21,16 @@ import org.graylog.security.certutil.CertConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Path;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -34,21 +38,15 @@ public class TruststoreCreator {
private static final Logger LOG = LoggerFactory.getLogger(TruststoreCreator.class);
private final Map<String, X509Certificate> rootCertificates;
private final RootCertificateFinder rootCertificateFinder;
public TruststoreCreator(RootCertificateFinder rootCertificateFinder) {
this.rootCertificateFinder = rootCertificateFinder;
this.rootCertificates = new LinkedHashMap<>();
}
private final Map<String, X509Certificate> rootCertificates = new LinkedHashMap<>();
public static TruststoreCreator newTruststore() {
return new TruststoreCreator(new RootCertificateFinder());
return new TruststoreCreator();
}
public TruststoreCreator addRootCert(final String name, KeystoreInformation keystoreInformation,
final String alias) throws IOException, GeneralSecurityException {
final X509Certificate rootCert = rootCertificateFinder.findRootCert(keystoreInformation.location(), keystoreInformation.password(), alias);
final X509Certificate rootCert = findRootCert(keystoreInformation.location(), keystoreInformation.password(), alias);
rootCertificates.put(name, rootCert);
return this;
}
@@ -67,4 +65,32 @@ public class TruststoreCreator {
}
return new KeystoreInformation(truststorePath, truststorePassword);
}
private static X509Certificate findRootCert(Path keystorePath,
char[] password,
final String alias) throws IOException, GeneralSecurityException {
final KeyStore keystore = loadKeystore(keystorePath, password);
final Certificate[] certs = keystore.getCertificateChain(alias);
return Arrays.stream(certs)
.filter(cert -> cert instanceof X509Certificate)
.map(cert -> (X509Certificate) cert)
.filter(cert -> isRootCaCertificate(cert) || certs.length == 1)//TODO: certs.length == 1 may be temporary, our merged does not create a proper cert chain, it seems
.findFirst()
.orElseThrow(() -> new KeyStoreException("Keystore does not contain root X509Certificate in the certificate chain!"));
}
private static boolean isRootCaCertificate(X509Certificate cert) {
return cert.getSubjectX500Principal().equals(cert.getIssuerX500Principal());
}
private static KeyStore loadKeystore(final Path keystorePath,
final char[] password) throws IOException, GeneralSecurityException {
KeyStore nodeKeystore = KeyStore.getInstance(CertConstants.PKCS12);
try (final FileInputStream is = new FileInputStream(keystorePath.toFile())) {
nodeKeystore.load(is, password);
}
return nodeKeystore;
}
}

6
data-node/src/main/java/org/graylog/datanode/opensearch/statemachine/OpensearchStateMachine.java
View File

@@ -28,7 +28,7 @@ import java.util.Set;
public class OpensearchStateMachine extends StateMachine<OpensearchState, OpensearchEvent> {
private final Logger log = LoggerFactory.getLogger(OpensearchStateMachine.class);
private static final Logger LOG = LoggerFactory.getLogger(OpensearchStateMachine.class);
/**
* How many times can the OS rest api call fail before we switch to the failed state
@@ -69,7 +69,7 @@ public class OpensearchStateMachine extends StateMachine<OpensearchState, Opense
// the startupFailuresCounter keeps track of failed REST status calls and allow failures during the
// startup period
config.configure(OpensearchState.STARTING)
.onEntry(process::start)
.onEntryFrom(OpensearchEvent.PROCESS_STARTED, process::start) // we don't want to re-trigger start from OpensearchEvent.HEALTH_CHECK_FAILED bellow
.permitDynamic(OpensearchEvent.HEALTH_CHECK_FAILED,
() -> startupFailuresCounter.failedTooManyTimes() ? OpensearchState.FAILED : OpensearchState.STARTING,
startupFailuresCounter::increment)
@@ -144,7 +144,7 @@ public class OpensearchStateMachine extends StateMachine<OpensearchState, Opense
try {
super.fire(trigger);
} catch (Exception e) {
log.error(e.getMessage());
LOG.error(e.getMessage());
super.fire(errorEvent);
}
}