datanode distribution provider refactoring (#16867)

* datanode distribution provider refactoring

* postpone opensearch distribution check till preflight checks

* fixd OpensearchBinPreflightCheck

data-node/src/main/java/org/graylog/datanode/Configuration.java

@@ -30,6 +30,7 @@ import com.github.joschi.jadconfig.validators.URIAbsoluteValidator;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.net.InetAddresses;
 import org.graylog.datanode.configuration.BaseConfiguration;
+import org.graylog.datanode.configuration.OpensearchDistributionProvider;
 import org.graylog2.plugin.Tools;
 import org.graylog2.shared.SuppressForbidden;
 import org.joda.time.DateTimeZone;
@@ -274,16 +275,6 @@ public class Configuration extends BaseConfiguration {
         }
     }
 
-    @ValidatorMethod
-    @SuppressWarnings("unused")
-    public void validateOpenSearchDistribution() throws ValidationException {
-        try {
-            OpensearchDistribution.detectInDirectory(Path.of(getOpensearchDistributionRoot()));
-        } catch (RuntimeException | IOException e) {
-            throw new ValidationException("Could not detect OpenSearch in " + getOpensearchDistributionRoot() + ", please check your installation. Error was: " + e.getMessage());
-        }
-    }
-
     public String getNodeIdFile() {
         return nodeIdFile;
     }

data-node/src/main/java/org/graylog/datanode/OpensearchDistribution.java

@@ -16,116 +16,16 @@
  */
 package org.graylog.datanode;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.graylog.datanode.configuration.OpensearchArchitecture;
 
 import javax.annotation.Nullable;
-import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 public record OpensearchDistribution(Path directory, String version, @Nullable String platform,
-                                     @Nullable String architecture) {
-
-    private static final Logger LOG = LoggerFactory.getLogger(OpensearchDistribution.class);
-    public static final Pattern FULL_NAME_PATTERN = Pattern.compile("opensearch-(.*)-(.+)-(.+)");
-    public static final Pattern SHORT_NAME_PATTERN = Pattern.compile("opensearch-(.*)");
+                                     @Nullable OpensearchArchitecture architecture) {
 
     public OpensearchDistribution(Path path, String version) {
         this(path, version, null, null);
     }
 
-    public static OpensearchDistribution detectInDirectory(Path directory) throws IOException {
-        final var osArch = System.getProperty("os.arch");
-        return detectInDirectory(directory, osArch);
-    }
-
-    static OpensearchDistribution detectInDirectory(Path rootDistDirectory, String osArch) throws IOException {
-        Objects.requireNonNull(rootDistDirectory, "Dist directory needs to be provided");
-
-        // if the base directory points directly to one opensearch distribution, we should return it directly.
-        // If the format doesn't fit, we'll look for opensearch distributions in this root directory.
-        final Optional<OpensearchDistribution> distDirectory = parse(rootDistDirectory);
-        return distDirectory.orElseGet(() -> detectInSubdirectory(rootDistDirectory, osArch));
-
-    }
-
-    private static OpensearchDistribution detectInSubdirectory(Path directory, String osArch) {
-
-        final var archCode = archCode(osArch);
-
-        final List<OpensearchDistribution> opensearchDistributions;
-        try (
-                var files = Files.list(directory);
-        ) {
-            opensearchDistributions = files
-                    .filter(Files::isDirectory)
-                    .flatMap(f -> parse(f).stream())
-                    .toList();
-        } catch (IOException e) {
-            throw createErrorMessage(directory, archCode, "Failed to list content of provided directory", e);
-        }
-
-        if (opensearchDistributions.isEmpty()) {
-            throw createErrorMessage(directory, archCode, "Could not detect any opensearch distribution");
-        }
-
-        LOG.info("Found following opensearch distributions: " + opensearchDistributions.stream().map(d -> d.directory().toAbsolutePath()).toList());
-
-        return findByArchitecture(opensearchDistributions, archCode)
-                .orElseGet(() -> findWithoutArchitecture(opensearchDistributions)
-                        .orElseThrow(() -> createErrorMessage(directory, archCode, "No Opensearch distribution found for your system architecture")));
-    }
-
-    private static IllegalArgumentException createErrorMessage(Path directory, String osArch, String message) {
-        return createErrorMessage(directory, osArch, message, null);
-    }
-
-
-    private static IllegalArgumentException createErrorMessage(Path directory, String archCode, String errorMessage, Exception cause) {
-        final String message = String.format(Locale.ROOT, "%s. Directory used for Opensearch detection: %s. Please configure opensearch_location to a directory that contains an opensearch distribution for your architecture %s. You can download Opensearch from https://opensearch.org/downloads.html . Please extract the downloaded distribution and point opensearch_location configuration option to that directory.", errorMessage, directory.toAbsolutePath(), archCode);
-        return new IllegalArgumentException(message, cause);
-    }
-
-    public static String archCode(final String osArch) {
-        return switch (osArch) {
-            case "amd64" -> "x64";
-            case "x86_64" -> "x64";
-            case "aarch64" -> "aarch64";
-            case "arm64" -> "aarch64";
-            default ->
-                    throw new UnsupportedOperationException("Unsupported OpenSearch distribution architecture: " + osArch);
-        };
-    }
-
-    private static Optional<OpensearchDistribution> findByArchitecture(List<OpensearchDistribution> available, String archCode) {
-        return available.stream()
-                .filter(d -> archCode.equals(d.architecture()))
-                .findAny();
-    }
-
-    private static Optional<OpensearchDistribution> findWithoutArchitecture(List<OpensearchDistribution> available) {
-        return available.stream().filter(d -> d.architecture() == null).findFirst();
-    }
-
-    private static Optional<OpensearchDistribution> parse(Path path) {
-        final String filename = path.getFileName().toString();
-        final Matcher matcher = FULL_NAME_PATTERN.matcher(filename);
-        if (matcher.matches()) {
-            return Optional.of(new OpensearchDistribution(path, matcher.group(1), matcher.group(2), matcher.group(3)));
-        } else {
-            final Matcher shortMatcher = SHORT_NAME_PATTERN.matcher(filename);
-            if (shortMatcher.matches()) {
-                return Optional.of(new OpensearchDistribution(path, shortMatcher.group(1)));
-            } else {
-                return Optional.empty();
-            }
-        }
-    }
 }

data-node/src/main/java/org/graylog/datanode/bindings/DatanodeConfigurationBindings.java

@@ -17,12 +17,15 @@
 package org.graylog.datanode.bindings;
 
 import com.google.inject.AbstractModule;
+import org.graylog.datanode.OpensearchDistribution;
 import org.graylog.datanode.configuration.DatanodeConfiguration;
 import org.graylog.datanode.configuration.DatanodeConfigurationProvider;
+import org.graylog.datanode.configuration.OpensearchDistributionProvider;
 
 public class DatanodeConfigurationBindings extends AbstractModule {
     @Override
     protected void configure() {
         bind(DatanodeConfiguration.class).toProvider(DatanodeConfigurationProvider.class);
+        bind(OpensearchDistribution.class).toProvider(OpensearchDistributionProvider.class);
     }
 }

data-node/src/main/java/org/graylog/datanode/bindings/GenericInitializerBindings.java

@@ -20,7 +20,6 @@ import com.google.common.util.concurrent.Service;
 import com.google.inject.AbstractModule;
 import com.google.inject.multibindings.Multibinder;
 import org.graylog.datanode.configuration.OpensearchConfigurationProvider;
-import org.graylog.datanode.initializers.JerseyService;
 import org.graylog.datanode.initializers.PeriodicalsService;
 import org.graylog.datanode.management.OpensearchProcess;
 import org.graylog.datanode.management.OpensearchProcessService;

data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchBinPreflightCheck.java

@@ -28,25 +28,27 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Optional;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 public class OpensearchBinPreflightCheck implements PreflightCheck {
 
-    private final Path opensearchDir;
+    private final Supplier<Path> opensearchDirSupplier;
 
     private static final Logger LOG = LoggerFactory.getLogger(OpensearchBinPreflightCheck.class);
 
     @Inject
     public OpensearchBinPreflightCheck(DatanodeConfiguration datanodeConfiguration) {
-        this(datanodeConfiguration.opensearchDistribution().directory());
+        this(() -> datanodeConfiguration.opensearchDistributionProvider().get().directory());
     }
 
-    public OpensearchBinPreflightCheck(Path opensearchBaseDirectory) {
-        this.opensearchDir = opensearchBaseDirectory;
+    public OpensearchBinPreflightCheck(Supplier<Path> opensearchBaseDirectory) {
+        this.opensearchDirSupplier = opensearchBaseDirectory;
     }
 
     @Override
     public void runCheck() throws PreflightCheckException {
+        final Path opensearchDir = opensearchDirSupplier.get();
         if (!Files.isDirectory(opensearchDir)) {
             throw new PreflightCheckException("Opensearch base directory " + opensearchDir + " doesn't exist!");
         }

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfiguration.java

@@ -24,7 +24,7 @@ import org.graylog2.security.IndexerJwtAuthTokenProvider;
  * config file or from the ENV properties.
  */
 public record DatanodeConfiguration(
-        OpensearchDistribution opensearchDistribution,
+        OpensearchDistributionProvider opensearchDistributionProvider,
         String nodeName,
         int processLogsBufferSize,
         IndexerJwtAuthTokenProvider indexerJwtAuthTokenProvider

data-node/src/main/java/org/graylog/datanode/configuration/DatanodeConfigurationProvider.java

@@ -18,14 +18,24 @@ package org.graylog.datanode.configuration;
 
 import org.graylog.datanode.Configuration;
 import org.graylog.datanode.OpensearchDistribution;
+import org.graylog.datanode.process.OpensearchConfiguration;
 import org.graylog2.plugin.Tools;
 import org.graylog2.security.IndexerJwtAuthTokenProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.inject.Provider;
 import javax.inject.Singleton;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 @Singleton
 public class DatanodeConfigurationProvider implements Provider<DatanodeConfiguration> {
@@ -33,8 +43,7 @@ public class DatanodeConfigurationProvider implements Provider<DatanodeConfigura
     private final DatanodeConfiguration datanodeConfiguration;
 
     @Inject
-    public DatanodeConfigurationProvider(final Configuration localConfiguration, IndexerJwtAuthTokenProvider jwtTokenProvider) throws IOException {
-        final OpensearchDistribution opensearchDistribution = detectOpensearchDistribution(localConfiguration);
+    public DatanodeConfigurationProvider(final Configuration localConfiguration, IndexerJwtAuthTokenProvider jwtTokenProvider, OpensearchDistributionProvider opensearchDistribution) throws IOException {
         datanodeConfiguration = new DatanodeConfiguration(
                 opensearchDistribution,
                 getNodesFromConfig(localConfiguration.getDatanodeNodeName()),
@@ -53,7 +62,4 @@ public class DatanodeConfigurationProvider implements Provider<DatanodeConfigura
         return datanodeConfiguration;
     }
 
-    private static OpensearchDistribution detectOpensearchDistribution(final Configuration configuration) throws IOException {
-        return OpensearchDistribution.detectInDirectory(Path.of(configuration.getOpensearchDistributionRoot()));
-    }
 }

data-node/src/main/java/org/graylog/datanode/configuration/OpensearchArchitecture.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.configuration;
+
+public enum OpensearchArchitecture {
+    x64,
+    aarch64;
+
+    public static OpensearchArchitecture fromOperatingSystem() {
+        final var osArch = System.getProperty("os.arch");
+        return fromCode(osArch);
+    }
+
+    public static OpensearchArchitecture fromCode(String osArch) {
+        return switch (osArch) {
+            case "amd64" -> x64;
+            case "x86_64" -> x64;
+            case "x64" -> x64;
+            case "aarch64" -> aarch64;
+            case "arm64" -> aarch64;
+            default ->
+                    throw new UnsupportedOperationException("Unsupported OpenSearch distribution architecture: " + osArch);
+        };
+    }
+}

data-node/src/main/java/org/graylog/datanode/configuration/OpensearchConfigurationProvider.java

@@ -117,7 +117,7 @@ public class OpensearchConfigurationProvider implements Provider<OpensearchConfi
             }
 
             return new OpensearchConfiguration(
-                    datanodeConfiguration.opensearchDistribution().directory(),
+                    datanodeConfiguration.opensearchDistributionProvider().get().directory(),
                     opensearchConfigLocation,
                     localConfiguration.getBindAddress(),
                     localConfiguration.getHostname(),

data-node/src/main/java/org/graylog/datanode/configuration/OpensearchDistributionProvider.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.configuration;
+
+import com.google.common.base.Suppliers;
+import org.graylog.datanode.Configuration;
+import org.graylog.datanode.OpensearchDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Singleton
+public class OpensearchDistributionProvider implements Provider<OpensearchDistribution> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(OpensearchDistributionProvider.class);
+    public static final Pattern FULL_NAME_PATTERN = Pattern.compile("opensearch-(.*)-(.+)-(.+)");
+    public static final Pattern SHORT_NAME_PATTERN = Pattern.compile("opensearch-(.*)");
+
+    private final Supplier<OpensearchDistribution> distribution;
+
+    @Inject
+    public OpensearchDistributionProvider(final Configuration localConfiguration) {
+        this(Path.of(localConfiguration.getOpensearchDistributionRoot()), OpensearchArchitecture.fromOperatingSystem());
+    }
+
+    OpensearchDistributionProvider(final Path opensearchDistributionRoot, OpensearchArchitecture architecture) {
+        this.distribution = Suppliers.memoize(() -> detectInDirectory(opensearchDistributionRoot, architecture));
+    }
+
+    @Override
+    public OpensearchDistribution get() {
+        return distribution.get();
+    }
+
+    private static OpensearchDistribution detectInDirectory(Path rootDistDirectory, OpensearchArchitecture osArch) {
+        Objects.requireNonNull(rootDistDirectory, "Dist directory needs to be provided");
+
+        // if the base directory points directly to one opensearch distribution, we should return it directly.
+        // If the format doesn't fit, we'll look for opensearch distributions in this root directory.
+        final Optional<OpensearchDistribution> distDirectory = parse(rootDistDirectory);
+        return distDirectory.orElseGet(() -> detectInSubdirectory(rootDistDirectory, osArch));
+
+    }
+
+    private static OpensearchDistribution detectInSubdirectory(Path directory, OpensearchArchitecture arch) {
+        final List<OpensearchDistribution> opensearchDistributions;
+        try (
+                var files = Files.list(directory);
+        ) {
+            opensearchDistributions = files
+                    .filter(Files::isDirectory)
+                    .flatMap(f -> parse(f).stream())
+                    .toList();
+        } catch (IOException e) {
+            throw createErrorMessage(directory, arch, "Failed to list content of provided directory", e);
+        }
+
+        if (opensearchDistributions.isEmpty()) {
+            throw createErrorMessage(directory, arch, "Could not detect any opensearch distribution");
+        }
+
+        LOG.info("Found following opensearch distributions: " + opensearchDistributions.stream().map(d -> d.directory().toAbsolutePath()).toList());
+
+        return findByArchitecture(opensearchDistributions, arch)
+                .orElseGet(() -> findWithoutArchitecture(opensearchDistributions)
+                        .orElseThrow(() -> createErrorMessage(directory, arch, "No Opensearch distribution found for your system architecture")));
+    }
+
+    private static IllegalArgumentException createErrorMessage(Path directory, OpensearchArchitecture arch, String message) {
+        return createErrorMessage(directory, arch, message, null);
+    }
+
+
+    private static IllegalArgumentException createErrorMessage(Path directory, OpensearchArchitecture arch, String errorMessage, Exception cause) {
+        final String message = String.format(Locale.ROOT, "%s. Directory used for Opensearch detection: %s. Please configure opensearch_location to a directory that contains an opensearch distribution for your architecture %s. You can download Opensearch from https://opensearch.org/downloads.html . Please extract the downloaded distribution and point opensearch_location configuration option to that directory.", errorMessage, directory.toAbsolutePath(), arch);
+        return new IllegalArgumentException(message, cause);
+    }
+
+    public static String archCode(final String osArch) {
+        return switch (osArch) {
+            case "amd64" -> "x64";
+            case "x86_64" -> "x64";
+            case "aarch64" -> "aarch64";
+            case "arm64" -> "aarch64";
+            default ->
+                    throw new UnsupportedOperationException("Unsupported OpenSearch distribution architecture: " + osArch);
+        };
+    }
+
+    private static Optional<OpensearchDistribution> findByArchitecture(List<OpensearchDistribution> available, OpensearchArchitecture arch) {
+        return available.stream()
+                .filter(d -> arch.equals(d.architecture()))
+                .findAny();
+    }
+
+    private static Optional<OpensearchDistribution> findWithoutArchitecture(List<OpensearchDistribution> available) {
+        return available.stream().filter(d -> d.architecture() == null).findFirst();
+    }
+
+    private static Optional<OpensearchDistribution> parse(Path path) {
+        final String filename = path.getFileName().toString();
+        final Matcher matcher = FULL_NAME_PATTERN.matcher(filename);
+        if (matcher.matches()) {
+            return Optional.of(new OpensearchDistribution(path, matcher.group(1), matcher.group(2), OpensearchArchitecture.fromCode(matcher.group(3))));
+        } else {
+            final Matcher shortMatcher = SHORT_NAME_PATTERN.matcher(filename);
+            if (shortMatcher.matches()) {
+                return Optional.of(new OpensearchDistribution(path, shortMatcher.group(1)));
+            } else {
+                return Optional.empty();
+            }
+        }
+    }
+}

data-node/src/main/java/org/graylog/datanode/rest/StatusController.java

@@ -45,7 +45,7 @@ public class StatusController {
     public DataNodeStatus status() {
         return new DataNodeStatus(
                 version,
-                new StatusResponse(datanodeConfiguration.opensearchDistribution().version(), openSearch.processInfo())
+                new StatusResponse(datanodeConfiguration.opensearchDistributionProvider().get().version(), openSearch.processInfo())
         );
     }
 }