opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize Data Node Migration Wizard (#18812)

Browse Source 
* remove first directory compatibility check and steps, rename DIRECTORY_COMPATIBILITY_CHECK_PAGE2 to DIRECTORY_COMPATIBILITY_CHECK_PAGE

* skip ca and cert renewal steps if these exist
rename wrongly named methods

* skip provisioning steps in in-place and remote reindexing migration if data nodes have already been provisioned
only provision data nodes which haven't been provisioned yet

* fix in-place provisioning skip for provisioned and started data nodes

* add directory readable check to compatibility check
run compatibility check in migration trigger (not visible in UI yet)
add retry trigger on failed check
implement directoryCompatibilityCheckOK in MigrationActionsImpl
add directory check button label in UI

* add hostname to CompatibilityResult, run compatibility check for all data nodes on migration action, fix tests
This commit is contained in:
Matthias Oesterheld
2024-03-28 14:12:45 +01:00
committed by GitHub
parent c22071eb2c
commit 1855ca0014
18 changed files with 205 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data-node/src/main/java/org/graylog/datanode/Configuration.java
View File

@@ -86,7 +86,7 @@ public class Configuration extends BaseConfiguration {
@Parameter(value = "opensearch_location")
private String opensearchDistributionRoot = "dist";
@Parameter(value = "opensearch_data_location", required = true, validators = DirectoryReadableValidator.class)
@Parameter(value = "opensearch_data_location", required = true)
private Path opensearchDataLocation = Path.of("datanode/data");
@Parameter(value = "opensearch_logs_location", required = true, validators = DirectoryWritableValidator.class)

15
data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchDataDirCompatibilityCheck.java
View File

@@ -16,10 +16,13 @@
*/
package org.graylog.datanode.bootstrap.preflight;
import com.github.joschi.jadconfig.ValidationException;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import org.graylog.datanode.DirectoryReadableValidator;
import org.graylog.datanode.configuration.DatanodeConfiguration;
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
import org.graylog.datanode.filesystem.index.IndicesDirectoryParser;
import org.graylog.datanode.filesystem.index.dto.IndexInformation;
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
import org.graylog.datanode.filesystem.index.dto.NodeInformation;
import org.graylog.shaded.opensearch2.org.opensearch.Version;
@@ -28,13 +31,8 @@ import org.graylog2.bootstrap.preflight.PreflightCheckException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import java.nio.file.Path;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
public class OpensearchDataDirCompatibilityCheck implements PreflightCheck {
@@ -43,6 +41,8 @@ public class OpensearchDataDirCompatibilityCheck implements PreflightCheck {
private final boolean isFreshInstallation;
private final DatanodeConfiguration datanodeConfiguration;
private final IndicesDirectoryParser indicesDirectoryParser;
private final DirectoryReadableValidator directoryReadableValidator = new DirectoryReadableValidator();
@Inject
public OpensearchDataDirCompatibilityCheck(@Named("isFreshInstallation") boolean isFreshInstallation, DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) {
@@ -62,12 +62,15 @@ public class OpensearchDataDirCompatibilityCheck implements PreflightCheck {
final String opensearchVersion = datanodeConfiguration.opensearchDistributionProvider().get().version();
try {
directoryReadableValidator.validate(opensearchDataDir.toUri().toString(), opensearchDataDir);
final IndexerDirectoryInformation info = indicesDirectoryParser.parse(opensearchDataDir);
checkCompatibility(opensearchVersion, info);
final int indicesCount = info.nodes().stream().mapToInt(n -> n.indices().size()).sum();
LOG.info("Found {} indices and all of them are valid with current opensearch version {}", indicesCount, opensearchVersion);
} catch (IncompatibleIndexVersionException e) {
throw new PreflightCheckException("Index directory is not compatible with current version " + opensearchVersion + " of Opensearch, terminating.", e);
} catch (ValidationException e) {
throw new PreflightCheckException(e);
}
}

2
data-node/src/main/java/org/graylog/datanode/rest/CompatibilityResult.java
View File

@@ -18,7 +18,7 @@ package org.graylog.datanode.rest;
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
public record CompatibilityResult(String opensearchVersion,
public record CompatibilityResult(String hostname, String opensearchVersion,
IndexerDirectoryInformation info,
java.util.List<String> compatibilityErrors) {
}

29
data-node/src/main/java/org/graylog/datanode/rest/IndicesDirectoryController.java
View File

@@ -16,21 +16,21 @@
*/
package org.graylog.datanode.rest;
import com.github.joschi.jadconfig.ValidationException;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import org.graylog.datanode.Configuration;
import org.graylog.datanode.DirectoryReadableValidator;
import org.graylog.datanode.configuration.DatanodeConfiguration;
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
import org.graylog.datanode.filesystem.index.IndicesDirectoryParser;
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
import org.graylog.datanode.filesystem.index.dto.NodeInformation;
import org.graylog.datanode.rest.config.OnlyInSecuredNode;
import org.graylog.shaded.opensearch2.org.opensearch.Version;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
@@ -39,11 +39,14 @@ import java.util.Locale;
@Produces(MediaType.APPLICATION_JSON)
public class IndicesDirectoryController {
private final Configuration configuration;
private final DatanodeConfiguration datanodeConfiguration;
private final IndicesDirectoryParser indicesDirectoryParser;
private final DirectoryReadableValidator directoryReadableValidator = new DirectoryReadableValidator();
@Inject
public IndicesDirectoryController(DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) {
public IndicesDirectoryController(Configuration configuration, DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) {
this.configuration = configuration;
this.datanodeConfiguration = datanodeConfiguration;
this.indicesDirectoryParser = indicesDirectoryParser;
}
@@ -53,16 +56,18 @@ public class IndicesDirectoryController {
public CompatibilityResult status() {
final java.nio.file.Path dataTargetDir = datanodeConfiguration.datanodeDirectories().getDataTargetDir();
final String opensearchVersion = datanodeConfiguration.opensearchDistributionProvider().get().version();
final String hostname = configuration.getHostname();
try {
directoryReadableValidator.validate(dataTargetDir.toUri().toString(), dataTargetDir);
final IndexerDirectoryInformation info = indicesDirectoryParser.parse(dataTargetDir);
final Version currentVersion = Version.fromString(opensearchVersion);
final List<String> compatibilityErrors = info.nodes().stream()
.filter(node -> !isNodeCompatible(node, currentVersion))
.map(node -> String.format(Locale.ROOT, "Current version %s of Opensearch is not compatible with index version %s", currentVersion, node.nodeVersion()))
.toList();
return new CompatibilityResult(opensearchVersion, info, compatibilityErrors);
} catch (IncompatibleIndexVersionException e) {
return new CompatibilityResult(opensearchVersion, new IndexerDirectoryInformation(dataTargetDir, Collections.emptyList()), Collections.singletonList(e.getMessage()));
return new CompatibilityResult(hostname, opensearchVersion, info, compatibilityErrors);
} catch (IncompatibleIndexVersionException | ValidationException e) {
return new CompatibilityResult(hostname, opensearchVersion, new IndexerDirectoryInformation(dataTargetDir, Collections.emptyList()), Collections.singletonList(e.getMessage()));
}
}