diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java index c78a7edb26..ddfb23af09 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java @@ -18,7 +18,8 @@ package org.graylog.datanode.bindings; import com.google.inject.AbstractModule; import com.google.inject.multibindings.MapBinder; -import org.graylog.datanode.bootstrap.OpensearchBinPreflightCheck; +import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck; +import org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync; import org.graylog2.bootstrap.preflight.PreflightCheck; public class PreflightChecksBindings extends AbstractModule { @@ -26,6 +27,7 @@ public class PreflightChecksBindings extends AbstractModule { @Override protected void configure() { + addPreflightCheck(OpensearchConfigSync.class); addPreflightCheck(OpensearchBinPreflightCheck.class); } diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java new file mode 100644 index 0000000000..720404ace0 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/FullDirSync.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.bootstrap.preflight; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.LinkedList; +import java.util.List; + +public class FullDirSync { + private static final Logger LOG = LoggerFactory.getLogger(FullDirSync.class); + + public static void run(Path source, Path target) throws IOException { + + final List existingPaths = collectExistingPaths(target); + final List copiedPaths = copyFiles(source, target); + + existingPaths.removeAll(copiedPaths); + + for (Path path : existingPaths) { + LOG.info("Deleting obsolete file " + path); + if (Files.isDirectory(path)) { + FileUtils.deleteDirectory(path.toFile()); + } else { + Files.deleteIfExists(path); + } + } + } + + private static List copyFiles(Path source, Path target) throws IOException { + + List copiedPaths = new LinkedList<>(); + + Files.walkFileTree(source, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path currentTarget = target.resolve(source.relativize(dir).toString()); + Files.createDirectories(currentTarget); + copiedPaths.add(currentTarget); + LOG.info("Synchronizing directory {}", currentTarget); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + final Path currentTarget = target.resolve(source.relativize(file).toString()); + Files.copy(file, currentTarget, StandardCopyOption.REPLACE_EXISTING); + copiedPaths.add(currentTarget); + LOG.info("Synchronizing file {}", currentTarget); + return FileVisitResult.CONTINUE; + } + }); + + return copiedPaths; + } + + private static List collectExistingPaths(Path target) throws IOException { + final List pathsToDelete = new LinkedList<>(); + + Files.walkFileTree(target, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + pathsToDelete.add(dir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + pathsToDelete.add(file); + return FileVisitResult.CONTINUE; + } + }); + return pathsToDelete; + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheck.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchBinPreflightCheck.java similarity index 95% rename from data-node/src/main/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheck.java rename to data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchBinPreflightCheck.java index c0f541bd16..dc7afdec2d 100644 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheck.java +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchBinPreflightCheck.java @@ -14,7 +14,7 @@ * along with this program. If not, see * . */ -package org.graylog.datanode.bootstrap; +package org.graylog.datanode.bootstrap.preflight; import org.graylog.datanode.OpensearchDistribution; import org.graylog2.bootstrap.preflight.PreflightCheck; @@ -41,7 +41,7 @@ public class OpensearchBinPreflightCheck implements PreflightCheck { this(opensearchDistribution.directory()); } - protected OpensearchBinPreflightCheck(Path opensearchBaseDirectory) { + public OpensearchBinPreflightCheck(Path opensearchBaseDirectory) { this.opensearchDir = opensearchBaseDirectory; } diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java new file mode 100644 index 0000000000..47b7f403b3 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchConfigSync.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.bootstrap.preflight; + +import org.apache.logging.log4j.core.appender.rolling.action.DeletingVisitor; +import org.apache.logging.log4j.core.appender.rolling.action.PathCondition; +import org.graylog.datanode.Configuration; +import org.graylog2.bootstrap.preflight.PreflightCheck; +import org.graylog2.bootstrap.preflight.PreflightCheckException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; + +public class OpensearchConfigSync implements PreflightCheck { + + private static final Logger LOG = LoggerFactory.getLogger(OpensearchConfigSync.class); + + private final Configuration configuration; + + @Inject + public OpensearchConfigSync(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public void runCheck() throws PreflightCheckException { + final Path localOpensearchConfigDir = Path.of(configuration.getOpensearchConfigLocation()).resolve("opensearch"); + LOG.info("Directory used for Opensearch configuration is {}", localOpensearchConfigDir.toAbsolutePath()); + + try { + Files.createDirectories(localOpensearchConfigDir); + synchronizeConfig(Path.of("opensearch", "config"), localOpensearchConfigDir); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException("Failed to prepare opensearch config directory", e); + + } + } + + public void synchronizeConfig(Path configRelativePath, final Path target) throws URISyntaxException, IOException { + final URI uriToConfig = OpensearchConfigSync.class.getResource("/" + configRelativePath.toString()).toURI(); + if ("jar".equals(uriToConfig.getScheme())) { + copyFromJar(configRelativePath, target, uriToConfig); + } else { + copyFromLocalFs(configRelativePath, target); + } + } + + private static void copyFromJar(Path configRelativePath, Path target, URI uri) throws IOException { + try ( + final FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap()); + ) { + // Get hold of the path to the top level directory of the JAR file + final Path resourcesRoot = fs.getPath("/"); + final Path source = resourcesRoot.resolve(configRelativePath.toString()); // caution, the toString is needed here to resolve properly! + copyRecursively(source, target); + } + } + + private static void copyFromLocalFs(Path configRelativePath, Path target) throws URISyntaxException, IOException { + final Path resourcesRoot = Paths.get(OpensearchConfigSync.class.getResource("/").toURI()); + final Path source = resourcesRoot.resolve(configRelativePath); + copyRecursively(source, target); + } + + private static void copyRecursively(Path source, Path target) throws IOException { + LOG.info("Synchronizing Opensearch configuration"); + FullDirSync.run(source, target); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/ConfigurationProvider.java b/data-node/src/main/java/org/graylog/datanode/configuration/ConfigurationProvider.java index 31812669ae..f7f9f7763c 100644 --- a/data-node/src/main/java/org/graylog/datanode/configuration/ConfigurationProvider.java +++ b/data-node/src/main/java/org/graylog/datanode/configuration/ConfigurationProvider.java @@ -59,7 +59,7 @@ public class ConfigurationProvider implements Provider public ConfigurationProvider(Configuration localConfiguration, DataNodeConfig sharedConfiguration, OpensearchDistribution opensearchDistribution) throws KeyStoreException, CertificateException, IOException, NoSuchAlgorithmException, UnrecoverableKeyException { final var cfg = sharedConfiguration.test(); - final Path opensearchConfigDir = opensearchDistribution.directory().resolve("config"); + final Path opensearchConfigDir = Path.of(localConfiguration.getOpensearchConfigLocation()).resolve("opensearch"); final LinkedHashMap config = new LinkedHashMap<>(); config.put("path.data", Path.of(localConfiguration.getOpensearchDataLocation()).resolve(localConfiguration.getDatanodeNodeName()).toAbsolutePath().toString()); @@ -155,6 +155,7 @@ public class ConfigurationProvider implements Provider configuration = new OpensearchConfiguration( opensearchDistribution.version(), opensearchDistribution.directory(), + Path.of(localConfiguration.getOpensearchConfigLocation()), localConfiguration.getOpensearchHttpPort(), localConfiguration.getOpensearchTransportPort(), localConfiguration.getRestApiUsername(), diff --git a/data-node/src/main/java/org/graylog/datanode/management/CommandLineProcess.java b/data-node/src/main/java/org/graylog/datanode/management/CommandLineProcess.java index b1bb9e730e..a7f7c51bc2 100644 --- a/data-node/src/main/java/org/graylog/datanode/management/CommandLineProcess.java +++ b/data-node/src/main/java/org/graylog/datanode/management/CommandLineProcess.java @@ -16,7 +16,6 @@ */ package org.graylog.datanode.management; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.exec.PumpStreamHandler; @@ -27,15 +26,10 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static java.util.function.Predicate.not; class CommandLineProcess { @@ -44,27 +38,20 @@ class CommandLineProcess { private final Path executable; private final List arguments; private final ProcessListener listener; - private final Supplier executorSupplier; - private final Supplier> environmentSupplier; + private final Environment environment; private final ExecuteWatchdog watchDog; private Process process; - public CommandLineProcess(Path executable, List arguments, ProcessListener listener) { - this(executable, arguments, listener, ProcessProvidingExecutor::new, System::getenv); - } - @VisibleForTesting CommandLineProcess(Path executable, List arguments, ProcessListener listener, - Supplier executorSupplier, - Supplier> environmentSupplier) { + Environment environment) { this.executable = executable; this.arguments = arguments; this.listener = listener; - this.environmentSupplier = environmentSupplier; - this.executorSupplier = executorSupplier; + this.environment = environment; this.watchDog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT); } @@ -78,12 +65,12 @@ class CommandLineProcess { arguments.forEach(it -> cmdLine.addArgument(it, true)); - ProcessProvidingExecutor executor = executorSupplier.get(); + ProcessProvidingExecutor executor = new ProcessProvidingExecutor(); - //executor.setStreamHandler(logs); executor.setStreamHandler(new PumpStreamHandler(new LoggingOutputStream(listener::onStdOut), new LoggingOutputStream(listener::onStdErr))); executor.setWatchdog(watchDog); - executor.execute(cmdLine, cleanEnvironment(environmentSupplier.get()), listener); + + executor.execute(cmdLine, environment.getEnv(), listener); try { this.process = executor.getProcess().get(30, TimeUnit.SECONDS); listener.onStart(); @@ -94,13 +81,6 @@ class CommandLineProcess { } } - private Map cleanEnvironment(Map env) { - return env.entrySet().stream() - // Remove JAVA_HOME from environment because OpenSearch should use its bundled JVM. - .filter(not(entry -> JAVA_HOME_ENV.equals(entry.getKey()))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - public void stop() { this.watchDog.destroyProcess(); } diff --git a/data-node/src/main/java/org/graylog/datanode/management/Environment.java b/data-node/src/main/java/org/graylog/datanode/management/Environment.java new file mode 100644 index 0000000000..361e98f4a7 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/management/Environment.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.management; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.function.Predicate.not; + +public class Environment { + + private static final String JAVA_HOME_ENV = "JAVA_HOME"; + + private final Map env; + private final Map additionalVariables = new HashMap<>(); + + public Environment(Map env) { + this.env = env; + } + + public Environment put(String key, String value) { + this.additionalVariables.put(key, value); + return this; + } + + public Map getEnv() { + Map env = new HashMap<>(); + env.putAll(cleanEnvironment(env)); + env.putAll(additionalVariables); + return Collections.unmodifiableMap(env); + } + + private Map cleanEnvironment(Map env) { + return env.entrySet().stream() + // Remove JAVA_HOME from environment because OpenSearch should use its bundled JVM. + .filter(not(entry -> JAVA_HOME_ENV.equals(entry.getKey()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } +} + diff --git a/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java b/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java index 95a9bfb466..55de789a0f 100644 --- a/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java +++ b/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java @@ -19,6 +19,7 @@ package org.graylog.datanode.management; import com.github.oxo42.stateless4j.StateMachine; import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.commons.exec.ExecuteException; +import org.graylog.datanode.ProcessProvidingExecutor; import org.graylog.datanode.process.OpensearchConfiguration; import org.graylog.datanode.process.ProcessEvent; import org.graylog.datanode.process.ProcessInfo; @@ -39,6 +40,7 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -155,7 +157,7 @@ class OpensearchProcessImpl implements OpensearchProcess, ProcessListener { final Path executable = configuration.opensearchDir().resolve(Paths.get("bin", "opensearch")); final List arguments = configuration.asMap().entrySet().stream() .map(it -> String.format(Locale.ROOT, "-E%s=%s", it.getKey(), it.getValue())).toList(); - commandLineProcess = new CommandLineProcess(executable, arguments, this); + commandLineProcess = new CommandLineProcess(executable, arguments, this, configuration.getEnv()); commandLineProcess.start(); } diff --git a/data-node/src/main/java/org/graylog/datanode/process/OpensearchConfiguration.java b/data-node/src/main/java/org/graylog/datanode/process/OpensearchConfiguration.java index ab1b2dee40..5e5cf60a73 100644 --- a/data-node/src/main/java/org/graylog/datanode/process/OpensearchConfiguration.java +++ b/data-node/src/main/java/org/graylog/datanode/process/OpensearchConfiguration.java @@ -16,6 +16,8 @@ */ package org.graylog.datanode.process; +import org.graylog.datanode.management.Environment; + import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.List; @@ -24,6 +26,7 @@ import java.util.Map; public record OpensearchConfiguration( String opensearchVersion, Path opensearchDir, + Path opensearchConfigDir, int httpPort, int transportPort, String authUsername, @@ -55,4 +58,10 @@ public record OpensearchConfiguration( private String toValuesList(List values) { return String.join(",", values); } + + public Environment getEnv() { + final Environment env = new Environment(System.getenv()); + env.put("OPENSEARCH_PATH_CONF", opensearchConfigDir.resolve("opensearch").toAbsolutePath().toString()); + return env; + } } diff --git a/data-node/src/main/resources/opensearch/config/jvm.options b/data-node/src/main/resources/opensearch/config/jvm.options new file mode 100644 index 0000000000..13fca96bad --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/jvm.options @@ -0,0 +1,80 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://opensearch.org/docs/opensearch/install/important-settings/ +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms1g +-Xmx1g + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +8-10:-XX:+UseConcMarkSweepGC +8-10:-XX:CMSInitiatingOccupancyFraction=75 +8-10:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1 GC is only supported on JDK version 10 or later +# to use G1GC, uncomment the next two lines and update the version on the +# following three lines to your version of the JDK +# 10:-XX:-UseConcMarkSweepGC +# 10:-XX:-UseCMSInitiatingOccupancyOnly +11-:-XX:+UseG1GC +11-:-XX:G1ReservePercent=25 +11-:-XX:InitiatingHeapOccupancyPercent=30 + +## JVM temporary directory +-Djava.io.tmpdir=${OPENSEARCH_TMPDIR} + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=data + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile=/tmp/hs_err_pid%p.log + +## JDK 8 GC logging +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:/tmp/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file=/tmp/gc.log:utctime,pid,tags:filecount=32,filesize=64m + +# Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380) +18-:-Djava.security.manager=allow diff --git a/data-node/src/main/resources/opensearch/config/log4j2.properties b/data-node/src/main/resources/opensearch/config/log4j2.properties new file mode 100644 index 0000000000..bb27aaf2e2 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/log4j2.properties @@ -0,0 +1,234 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +######## Server JSON ############################ +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_server.json +appender.rolling.filePermissions = rw-r----- +appender.rolling.layout.type = OpenSearchJsonLayout +appender.rolling.layout.type_name = server + +appender.rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:opensearch.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:opensearch.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ +######## Server - old style pattern ########### +appender.rolling_old.type = RollingFile +appender.rolling_old.name = rolling_old +appender.rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}.log +appender.rolling_old.filePermissions = rw-r----- +appender.rolling_old.layout.type = PatternLayout +appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_old.policies.type = Policies +appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_old.policies.time.interval = 1 +appender.rolling_old.policies.time.modulate = true +appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_old.policies.size.size = 128MB +appender.rolling_old.strategy.type = DefaultRolloverStrategy +appender.rolling_old.strategy.fileIndex = nomax +appender.rolling_old.strategy.action.type = Delete +appender.rolling_old.strategy.action.basepath = ${sys:opensearch.logs.base_path} +appender.rolling_old.strategy.action.condition.type = IfFileName +appender.rolling_old.strategy.action.condition.glob = ${sys:opensearch.logs.cluster_name}-* +appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling +rootLogger.appenderRef.rolling_old.ref = rolling_old + +######## Deprecation JSON ####################### +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation.json +appender.deprecation_rolling.filePermissions = rw-r----- +appender.deprecation_rolling.layout.type = OpenSearchJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation +appender.deprecation_rolling.layout.opensearchmessagefields=x-opaque-id +appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter + +appender.deprecation_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation-%i.json.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +appender.header_warning.type = HeaderWarningAppender +appender.header_warning.name = header_warning +################################################# +######## Deprecation - old style pattern ####### +appender.deprecation_rolling_old.type = RollingFile +appender.deprecation_rolling_old.name = deprecation_rolling_old +appender.deprecation_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_deprecation.log +appender.deprecation_rolling_old.filePermissions = rw-r----- +appender.deprecation_rolling_old.layout.type = PatternLayout +appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.deprecation_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _deprecation-%i.log.gz +appender.deprecation_rolling_old.policies.type = Policies +appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling_old.policies.size.size = 1GB +appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling_old.strategy.max = 4 +################################################# +logger.deprecation.name = org.opensearch.deprecation +logger.deprecation.level = deprecation +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old +logger.deprecation.appenderRef.header_warning.ref = header_warning +logger.deprecation.additivity = false + +######## Search slowlog JSON #################### +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\ + .cluster_name}_index_search_slowlog.json +appender.index_search_slowlog_rolling.filePermissions = rw-r----- +appender.index_search_slowlog_rolling.layout.type = OpenSearchJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog +appender.index_search_slowlog_rolling.layout.opensearchmessagefields=message,took,took_millis,total_hits,types,stats,search_type,total_shards,source,id + +appender.index_search_slowlog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs\ + .cluster_name}_index_search_slowlog-%i.json.gz +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.size.size = 1GB +appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling.strategy.max = 4 +################################################# +######## Search slowlog - old style pattern #### +appender.index_search_slowlog_rolling_old.type = RollingFile +appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old +appender.index_search_slowlog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_search_slowlog.log +appender.index_search_slowlog_rolling_old.filePermissions = rw-r----- +appender.index_search_slowlog_rolling_old.layout.type = PatternLayout +appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.index_search_slowlog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_search_slowlog-%i.log.gz +appender.index_search_slowlog_rolling_old.policies.type = Policies +appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling_old.policies.size.size = 1GB +appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling_old.strategy.max = 4 +################################################# +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old +logger.index_search_slowlog_rolling.additivity = false + +######## Indexing slowlog JSON ################## +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_indexing_slowlog.json +appender.index_indexing_slowlog_rolling.filePermissions = rw-r----- +appender.index_indexing_slowlog_rolling.layout.type = OpenSearchJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog +appender.index_indexing_slowlog_rolling.layout.opensearchmessagefields=message,took,took_millis,doc_type,id,routing,source + +appender.index_indexing_slowlog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_indexing_slowlog-%i.json.gz +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling.strategy.max = 4 +################################################# +######## Indexing slowlog - old style pattern ## +appender.index_indexing_slowlog_rolling_old.type = RollingFile +appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old +appender.index_indexing_slowlog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling_old.filePermissions = rw-r----- +appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}\ + _index_indexing_slowlog-%i.log.gz +appender.index_indexing_slowlog_rolling_old.policies.type = Policies +appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling_old.strategy.max = 4 +################################################# + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old +logger.index_indexing_slowlog.additivity = false + +######## Task details log JSON #################### +appender.task_detailslog_rolling.type = RollingFile +appender.task_detailslog_rolling.name = task_detailslog_rolling +appender.task_detailslog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.json +appender.task_detailslog_rolling.filePermissions = rw-r----- +appender.task_detailslog_rolling.layout.type = OpenSearchJsonLayout +appender.task_detailslog_rolling.layout.type_name = task_detailslog +appender.task_detailslog_rolling.layout.opensearchmessagefields=taskId,type,action,description,start_time_millis,resource_stats,metadata + +appender.task_detailslog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.json.gz +appender.task_detailslog_rolling.policies.type = Policies +appender.task_detailslog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.task_detailslog_rolling.policies.size.size = 1GB +appender.task_detailslog_rolling.strategy.type = DefaultRolloverStrategy +appender.task_detailslog_rolling.strategy.max = 4 +################################################# +######## Task details log - old style pattern #### +appender.task_detailslog_rolling_old.type = RollingFile +appender.task_detailslog_rolling_old.name = task_detailslog_rolling_old +appender.task_detailslog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.log +appender.task_detailslog_rolling_old.filePermissions = rw-r----- +appender.task_detailslog_rolling_old.layout.type = PatternLayout +appender.task_detailslog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.task_detailslog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.log.gz +appender.task_detailslog_rolling_old.policies.type = Policies +appender.task_detailslog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.task_detailslog_rolling_old.policies.size.size = 1GB +appender.task_detailslog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.task_detailslog_rolling_old.strategy.max = 4 +################################################# +logger.task_detailslog_rolling.name = task.detailslog +logger.task_detailslog_rolling.level = trace +logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling +logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling_old.ref = task_detailslog_rolling_old +logger.task_detailslog_rolling.additivity = false diff --git a/data-node/src/main/resources/opensearch/config/opensearch-notifications-core/notifications-core.yml b/data-node/src/main/resources/opensearch/config/opensearch-notifications-core/notifications-core.yml new file mode 100644 index 0000000000..63f21b1418 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-notifications-core/notifications-core.yml @@ -0,0 +1,19 @@ +--- +## + # Copyright OpenSearch Contributors + # SPDX-License-Identifier: Apache-2.0 +## + +# configuration file for the notifications-core plugin +opensearch.notifications.core: + email: + size_limit: 10000 + minimum_header_length: 100 + http: + max_connections: 60 + max_connection_per_route: 20 + connection_timeout: 5000 # in milliseconds + socket_timeout: 50000 + host_deny_list: [] + allowed_config_types: ["slack","chime","webhook","email","sns","ses_account","smtp_account","email_group"] + tooltip_support: true diff --git a/data-node/src/main/resources/opensearch/config/opensearch-notifications/notifications.yml b/data-node/src/main/resources/opensearch/config/opensearch-notifications/notifications.yml new file mode 100644 index 0000000000..61969acfb3 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-notifications/notifications.yml @@ -0,0 +1,12 @@ +--- +## + # Copyright OpenSearch Contributors + # SPDX-License-Identifier: Apache-2.0 +## + +# configuration file for the notifications plugin +opensearch.notifications: + general: + operation_timeout_ms: 60000 # 60 seconds, Minimum 100ms + default_items_query_count: 100 # default number of items to query + filter_send_by_backend_roles: false # Does sendNotification needs to validate user's backend roles diff --git a/data-node/src/main/resources/opensearch/config/opensearch-observability/observability.yml b/data-node/src/main/resources/opensearch/config/opensearch-observability/observability.yml new file mode 100644 index 0000000000..6bc2c48720 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-observability/observability.yml @@ -0,0 +1,28 @@ +--- +## +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +## + +# configuration file for the observability plugin +opensearch.notebooks: + general: + operationTimeoutMs: 60000 # 60 seconds, Minimum 100ms + defaultItemsQueryCount: 100 # default number of items to query + polling: + jobLockDurationSeconds: 300 # 5 Minutes, Minimum 10 seconds + minPollingDurationSeconds: 300 # 5 Minutes, Minimum 60 seconds + maxPollingDurationSeconds: 900 # 15 Minutes, Minimum 5 Minutes + maxLockRetries: 1 # Max number of retries to retry locking + access: + adminAccess: "AllObservabilityObjects" + # adminAccess values: + ## Standard -> Admin user access follows standard user + ## AllObservabilityObjects -> Admin user with "all_access" role can see all observability objects of all users. + filterBy: "NoFilter" # Applied when tenant != __user__ + # filterBy values: + ## NoFilter -> everyone see each other's observability objects + ## User -> observability objects are visible to only themselves + ## Roles -> observability objects are visible to users having any one of the role of creator + ## BackendRoles -> observability objects are visible to users having any one of the backend role of creator + ignoreRoles: ["own_index", "kibana_user", "observability_full_access", "observability_read_access"] diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/agent-stats-metadata b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/agent-stats-metadata new file mode 100644 index 0000000000..a00fccece4 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/agent-stats-metadata @@ -0,0 +1,2 @@ +# All the meta data to be emitted in the agent stats; an example below +Program=PerformanceAnalyzerAgent diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/log4j2.xml b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/log4j2.xml new file mode 100644 index 0000000000..9d14c33e7f --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/log4j2.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy new file mode 100644 index 0000000000..365598b6e5 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy @@ -0,0 +1,16 @@ +grant { + permission java.lang.management.ManagementPermission "control"; + permission java.net.SocketPermission "localhost:9600","connect,resolve"; +}; + +grant codebase "file:${java.home}/../lib/tools.jar" { + permission java.security.AllPermission; +}; + +grant codeBase "jrt:/jdk.attach" { + permission java.security.AllPermission; +}; + +grant codeBase "jrt:/jdk.internal.jvmstat" { + permission java.security.AllPermission; +}; diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/performance-analyzer.properties b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/performance-analyzer.properties new file mode 100644 index 0000000000..3ef0461b46 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/performance-analyzer.properties @@ -0,0 +1,46 @@ +# ======================== OpenSearch performance analyzer plugin config ========================= + +# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS. + +# Metrics data location +metrics-location = /dev/shm/performanceanalyzer/ + +# Metrics deletion interval (minutes) for metrics data. +# Interval should be between 1 to 60. +metrics-deletion-interval = 1 + +# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2 +# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving +# the files and wouldn't like for them to be cleaned up. +cleanup-metrics-db-files = true + +# WebService exposed by App's port +webservice-listener-port = 9600 + +# Port for RPC Communication +rpc-port = 9650 + +# Metric DB File Prefix Path location +metrics-db-file-prefix-path = /tmp/metricsdb_ + +https-enabled = false + +# Setup the correct path for server certificates +certificate-file-path = specify_path +private-key-file-path = specify_path +#trusted-cas-file-path = specify_path + +# Setup the correct path for client certificates (by default, the client will just use the server certificates) +#client-certificate-file-path = specify_path +#client-private-key-file-path = specify_path +#client-trusted-cas-file-path = specify_path + +# WebService bind host; default only to local interface +#webservice-bind-host = + +# Plugin Stats Metadata file name, expected to be in the same location +plugin-stats-metadata = plugin-stats-metadata + +# Agent Stats Metadata file name, expected to be in the same location +agent-stats-metadata = agent-stats-metadata + diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata new file mode 100644 index 0000000000..d6de58dce4 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/plugin-stats-metadata @@ -0,0 +1,2 @@ +# All the meta data to be emitted in the plugin stats; an example below +Program=PerformanceAnalyzerPlugin diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca.conf b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca.conf new file mode 100644 index 0000000000..6122403cc2 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca.conf @@ -0,0 +1,105 @@ +{ + "analysis-graph-implementor": "org.opensearch.performanceanalyzer.rca.store.OpenSearchAnalysisGraph", + // it can be file:// or s3:// + "rca-store-location": "s3://sifi-store/rcas/", + + //it can be file:// or s3:// + "threshold-store-location": "s3://sifi-store/thresholds/", + + "new-rca-check-minutes": 60, + + "new-threshold-check-minutes": 30, + + // The size of the task queue for all networking operations. + // Small size queues may result in dropping of flow units, while large size queues can lead to a + // bigger backlog of tasks resulting in delays in sending and receiving. + "network-queue-length": 200, + + // The size of the per-vertex buffer for flow units received from remote nodes. + // Small buffer sizes may result in dropping of flow units, while large buffer size can lead to + // high memory consumptions depending on how the analysis graph is configured. + "max-flow-units-per-vertex-buffer": 200, + + "tags": { + "locus": "data-node" + }, + + "remote-peers": ["ip1", "ip2", "ip3"], + + // Tells the runtime where the RCAs will be stored. + "datastore": { + // accepted types are sqlite, in-memory. + "type": "sqlite", + "location-dir": "/tmp", + "filename": "rca.sqlite", + "storage-file-retention-count": 5, + // How often the sqlite file be repeated in seconds. This file contains RCAs and therefore rotating it too frequently + // might not be as fruitful as there might not be any data. + "rotation-period-seconds": 21600 + }, + + // Add config settings for different RCAs + "rca-config-settings": { + // old gen rca + "high-heap-usage-old-gen-rca": { + "top-k" : 3 + }, + //young gen rca + "high-heap-usage-young-gen-rca": { + "promotion-rate-mb-per-second" : 500, + "young-gen-gc-time-ms-per-second" : 400 + }, + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, + //hot shard rca + "hot-shard-rca": { + "cpu-utilization" : 0.01, + "io-total-throughput-in-bytes" : 250000.0, + "io-total-syscallrate-per-second" : 0.1 + }, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { + "shard-request-cache-threshold" : 0.9 + }, + "admission-control-rca": { + "request-size": { + "heap-range": [ + { + "lower-bound": 0, + "upper-bound": 80, + "threshold": 15.0 + }, + { + "lower-bound": 81, + "upper-bound": 90, + "threshold": 10.0 + }, + { + "lower-bound": 91, + "upper-bound": 100, + "threshold": 5.0 + } + ] + } + } + }, + + "muted-rcas": [ + "HotShardRca", + "HotShardClusterRca" + ], + "muted-deciders": [], + "muted-actions": [], + "bucketization": { + "old-gen": { + "UNDER_UTILIZED": 10.0, + "HEALTHY_WITH_BUFFER": 60.0, + "HEALTHY": 80.0 + } + } +} diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_cluster_manager.conf b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_cluster_manager.conf new file mode 100644 index 0000000000..57dca33621 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_cluster_manager.conf @@ -0,0 +1,142 @@ +{ + "analysis-graph-implementor": "org.opensearch.performanceanalyzer.rca.store.OpenSearchAnalysisGraph", + // it can be file:// or s3:// + "rca-store-location": "s3://sifi-store/rcas/", + + //it can be file:// or s3:// + "threshold-store-location": "s3://sifi-store/thresholds/", + + "new-rca-check-minutes": 60, + + "new-threshold-check-minutes": 30, + + // The size of the task queue for all networking operations. + // Small size queues may result in dropping of flow units, while large size queues can lead to a + // bigger backlog of tasks resulting in delays in sending and receiving. + "network-queue-length": 200, + + // The size of the per-vertex buffer for flow units received from remote nodes. + // Small buffer sizes may result in dropping of flow units, while large buffer size can lead to + // high memory consumptions depending on how the analysis graph is configured. + "max-flow-units-per-vertex-buffer": 200, + + "tags": { + "locus": "cluster_manager-node" + }, + + "remote-peers": ["ip1", "ip2", "ip3"], + + // Tells the runtime where the RCAs will be stored. + "datastore": { + // accepted types are sqlite, in-memory. + "type": "sqlite", + "location-dir": "/tmp", + "filename": "rca.sqlite", + "storage-file-retention-count": 5, + // How often the sqlite file be repeated in seconds. This file contains RCAs and therefore rotating it too frequently + // might not be as fruitful as there might not be any data. + "rotation-period-seconds": 21600 + }, + + // Add config settings for different RCAs + "rca-config-settings": { + // old gen rca + "high-heap-usage-old-gen-rca": { + "top-k" : 3 + }, + //young gen rca + "high-heap-usage-young-gen-rca": { + "promotion-rate-mb-per-second" : 500, + "young-gen-gc-time-ms-per-second" : 400 + }, + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, + //unbalanced node rca + "hot-node-cluster-rca": { + "unbalanced-resource-percentage" : 0.3, + "resource-usage-lower-bound-percentage" : 0.1 + }, + //hot shard rca + "hot-shard-rca": { + "cpu-utilization" : 0.01, + "io-total-throughput-in-bytes" : 250000.0, + "io-total-syscallrate-per-second" : 0.1 + }, + //hot shard cluster rca + "hot-shard-cluster-rca": { + "cpu-utilization-cluster-percentage" : 0.3, + "io-total-throughput-cluster-percentage" : 0.3, + "io-total-syscallrate-cluster-percentage" : 0.3 + }, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { + "shard-request-cache-threshold" : 0.9 + } + }, + + "muted-rcas": [ + "HotShardRca", + "HotShardClusterRca" + ], + "muted-deciders": [], + "muted-actions": [], + + // Default Values are added in the config specific file as well + // Modify them as well whenever changing the default values (DeciderConfig.java) + "decider-config-settings": { + "workload-type": { + "prefer-ingest": true, + "prefer-search": true + }, + // Decreasing order of priority for the type of cache which is expected to be consumed more. + // Priority order in the list goes from most used to the lease used cache type. + "cache-type": { + "priority-order": ["fielddata-cache", "shard-request-cache", "query-cache", "bitset-filter-cache"] + }, + "old-gen-decision-policy-config": { + "queue-bucket-size" : 20, + "old-gen-threshold-level-one" : 0.6, + "old-gen-threshold-level-two" : 0.75, + "old-gen-threshold-level-three" : 0.9 + } + }, + // Action Configurations + "action-config-settings": { + // Cache Max Size bounds are expressed as %age of JVM heap size + "cache-settings": { + "total-step-count" : 20, + "fielddata": { + "upper-bound": 0.4, + "lower-bound": 0.1 + }, + "shard-request": { + "upper-bound": 0.05, + "lower-bound": 0.01 + } + }, + // Queue Capacity bounds are expressed as absolute queue size + "queue-settings": { + "total-step-count" : 20, + "search": { + "upper-bound": 3000, + "lower-bound": 1000 + }, + "write": { + "upper-bound": 1000, + "lower-bound": 200 + } + } + }, + "bucketization": { + "old-gen": { + "UNDER_UTILIZED": 10.0, + "HEALTHY_WITH_BUFFER": 60.0, + "HEALTHY": 80.0 + } + } +} diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_idle_cluster_manager.conf b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_idle_cluster_manager.conf new file mode 100644 index 0000000000..0a8b44e883 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/rca_idle_cluster_manager.conf @@ -0,0 +1,142 @@ +{ + "analysis-graph-implementor": "org.opensearch.performanceanalyzer.rca.store.OpenSearchAnalysisGraph", + // it can be file:// or s3:// + "rca-store-location": "s3://sifi-store/rcas/", + + //it can be file:// or s3:// + "threshold-store-location": "s3://sifi-store/thresholds/", + + "new-rca-check-minutes": 60, + + "new-threshold-check-minutes": 30, + + // The size of the task queue for all networking operations. + // Small size queues may result in dropping of flow units, while large size queues can lead to a + // bigger backlog of tasks resulting in delays in sending and receiving. + "network-queue-length": 200, + + // The size of the per-vertex buffer for flow units received from remote nodes. + // Small buffer sizes may result in dropping of flow units, while large buffer size can lead to + // high memory consumptions depending on how the analysis graph is configured. + "max-flow-units-per-vertex-buffer": 200, + + "tags": { + "locus": "idle-cluster_manager-node" + }, + + "remote-peers": ["ip1", "ip2", "ip3"], + + // Tells the runtime where the RCAs will be stored. + "datastore": { + // accepted types are sqlite, in-memory. + "type": "sqlite", + "location-dir": "/tmp", + "filename": "rca.sqlite", + "storage-file-retention-count": 5, + // How often the sqlite file be repeated in seconds. This file contains RCAs and therefore rotating it too frequently + // might not be as fruitful as there might not be any data. + "rotation-period-seconds": 21600 + }, + + // Add config settings for different RCAs + "rca-config-settings": { + // old gen rca + "high-heap-usage-old-gen-rca": { + "top-k" : 3 + }, + //young gen rca + "high-heap-usage-young-gen-rca": { + "promotion-rate-mb-per-second" : 500, + "young-gen-gc-time-ms-per-second" : 400 + }, + "queue-rejection-rca": { + "rejection-time-period-in-seconds" : 300 + }, + //unbalanced node rca + "hot-node-cluster-rca": { + "unbalanced-resource-percentage" : 0.3, + "resource-usage-lower-bound-percentage" : 0.1 + }, + //hot shard rca + "hot-shard-rca": { + "cpu-utilization" : 0.01, + "io-total-throughput-in-bytes" : 250000.0, + "io-total-syscallrate-per-second" : 0.1 + }, + //hot shard cluster rca + "hot-shard-cluster-rca": { + "cpu-utilization-cluster-percentage" : 0.3, + "io-total-throughput-cluster-percentage" : 0.3, + "io-total-syscallrate-cluster-percentage" : 0.3 + }, + // field data cache rca + "field-data-cache-rca": { + "field-data-cache-size-threshold" : 0.8 + }, + // shard request cache rca + "shard-request-cache-rca": { + "shard-request-cache-threshold" : 0.9 + } + }, + + "muted-rcas": [ + "HotShardRca", + "HotShardClusterRca" + ], + "muted-deciders": [], + "muted-actions": [], + + // Default Values are added in the config specific file as well + // Modify them as well whenever changing the default values (DeciderConfig.java) + "decider-config-settings": { + "workload-type": { + "prefer-ingest": true, + "prefer-search": true + }, + // Decreasing order of priority for the type of cache which is expected to be consumed more. + // Priority order in the list goes from most used to the lease used cache type. + "cache-type": { + "priority-order": ["fielddata-cache", "shard-request-cache", "query-cache", "bitset-filter-cache"] + }, + "old-gen-decision-policy-config": { + "queue-bucket-size" : 20, + "old-gen-threshold-level-one" : 0.6, + "old-gen-threshold-level-two" : 0.75, + "old-gen-threshold-level-three" : 0.9 + } + }, + // Action Configurations + "action-config-settings": { + // Cache Max Size bounds are expressed as %age of JVM heap size + "cache-settings": { + "total-step-count" : 20, + "fielddata": { + "upper-bound": 0.4, + "lower-bound": 0.1 + }, + "shard-request": { + "upper-bound": 0.05, + "lower-bound": 0.01 + } + }, + // Queue Capacity bounds are expressed as absolute queue size + "queue-settings": { + "total-step-count" : 20, + "search": { + "upper-bound": 3000, + "lower-bound": 1000 + }, + "write": { + "upper-bound": 1000, + "lower-bound": 200 + } + } + }, + "bucketization": { + "old-gen": { + "UNDER_UTILIZED": 10.0, + "HEALTHY_WITH_BUFFER": 60.0, + "HEALTHY": 80.0 + } + } +} diff --git a/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/supervisord.conf b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/supervisord.conf new file mode 100644 index 0000000000..47162f7b48 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-performance-analyzer/supervisord.conf @@ -0,0 +1,32 @@ +; supervisor config file + +[unix_http_server] +file=/usr/share/supervisor/performance_analyzer/supervisord.sock +chmod=0770 + +[supervisord] +logfile=/usr/share/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) +pidfile=/usr/share/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=/usr/share/supervisord.sock + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = /etc/supervisor/conf.d/*.conf + +[program:performance_analyzer] +command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch +user=1000 diff --git a/data-node/src/main/resources/opensearch/config/opensearch-reports-scheduler/reports-scheduler.yml b/data-node/src/main/resources/opensearch/config/opensearch-reports-scheduler/reports-scheduler.yml new file mode 100644 index 0000000000..d48464cacb --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-reports-scheduler/reports-scheduler.yml @@ -0,0 +1,11 @@ +--- +## + # Copyright OpenSearch Contributors + # SPDX-License-Identifier: Apache-2.0 +## + +# configuration file for the reports scheduler plugin +opensearch.reports: + general: + operationTimeoutMs: 60000 # 60 seconds, Minimum 100ms + defaultItemsQueryCount: 100 # default number of items to query diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/action_groups.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/action_groups.yml new file mode 100644 index 0000000000..7c40612b83 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/action_groups.yml @@ -0,0 +1,3 @@ +_meta: + type: "actiongroups" + config_version: 2 diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/allowlist.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/allowlist.yml new file mode 100644 index 0000000000..e669557d7e --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/allowlist.yml @@ -0,0 +1,69 @@ +--- +_meta: + type: "allowlist" + config_version: 2 + +# Description: +# enabled - feature flag. +# if enabled is false, the allowlisting feature is removed. +# This is like removing the check that checks if an API is allowlisted. +# This is equivalent to continuing with the usual access control checks, and removing all the code that implements allowlisting. +# if enabled is true, then all users except SuperAdmin can access only the APIs in requests +# SuperAdmin can access all APIs. +# SuperAdmin is defined by the SuperAdmin certificate, which is configured in the opensearch.yml setting: plugins.security.authcz.admin_dn: +# Refer to the example setting in opensearch.yml.example, and the opendistro documentation to know more about configuring SuperAdmin. +# +# requests - map of allowlisted endpoints, and the allowlisted HTTP requests for those endpoints + +# Examples showing how to configure this yml file (make sure the _meta data from above is also there): +# Example 1: +# To enable allowlisting and allowlist GET /_cluster/settings +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# +# Example 2: +# If you want to allowlist multiple request methods for /_cluster/settings (GET,PUT): +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# - PUT +# +# Example 3: +# If you want to allowlist other APIs as well, for example GET /_cat/nodes, and GET /_cat/shards: +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# - PUT +# /_cat/nodes: +# - GET +# /_cat/shards: +# - GET +# +# Example 4: +# If you want to disable the allowlisting feature, set enabled to false. +# enabled: false +# requests: +# /_cluster/settings: +# - GET +# +#At this point, all APIs become allowlisted because the feature to allowlist is off, so requests is irrelevant. + + +#this name must be config +config: + enabled: false + requests: + /_cluster/settings: + - GET + /_cat/nodes: + - GET diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/audit.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/audit.yml new file mode 100644 index 0000000000..dcfbad8dd7 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/audit.yml @@ -0,0 +1,85 @@ +_meta: + type: "audit" + config_version: 2 + +config: + # enable/disable audit logging + enabled: true + + audit: + # Enable/disable REST API auditing + enable_rest: true + + # Categories to exclude from REST API auditing + disabled_rest_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + + # Enable/disable Transport API auditing + enable_transport: true + + # Categories to exclude from Transport API auditing + disabled_transport_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + + # Users to be excluded from auditing. Wildcard patterns are supported. Eg: + # ignore_users: ["test-user", "employee-*"] + ignore_users: + - kibanaserver + + # Requests to be excluded from auditing. Wildcard patterns are supported. Eg: + # ignore_requests: ["indices:data/read/*", "SearchRequest"] + ignore_requests: [] + + # Log individual operations in a bulk request + resolve_bulk_requests: false + + # Include the body of the request (if available) for both REST and the transport layer + log_request_body: true + + # Logs all indices affected by a request. Resolves aliases and wildcards/date patterns + resolve_indices: true + + # Exclude sensitive headers from being included in the logs. Eg: Authorization + exclude_sensitive_headers: true + + compliance: + # enable/disable compliance + enabled: true + + # Log updates to internal security changes + internal_config: true + + # Log external config files for the node + external_config: false + + # Log only metadata of the document for read events + read_metadata_only: true + + # Map of indexes and fields to monitor for read events. Wildcard patterns are supported for both index names and fields. Eg: + # read_watched_fields: { + # "twitter": ["message"] + # "logs-*": ["id", "attr*"] + # } + read_watched_fields: {} + + # List of users to ignore for read events. Wildcard patterns are supported. Eg: + # read_ignore_users: ["test-user", "employee-*"] + read_ignore_users: + - kibanaserver + + # Log only metadata of the document for write events + write_metadata_only: true + + # Log only diffs for document updates + write_log_diffs: false + + # List of indices to watch for write events. Wildcard patterns are supported + # write_watched_indices: ["twitter", "logs-*"] + write_watched_indices: [] + + # List of users to ignore for write events. Wildcard patterns are supported. Eg: + # write_ignore_users: ["test-user", "employee-*"] + write_ignore_users: + - kibanaserver diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/config.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/config.yml new file mode 100644 index 0000000000..59f5f05352 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/config.yml @@ -0,0 +1,247 @@ +--- + +# This is the main OpenSearch Security configuration file where authentication +# and authorization is defined. +# +# You need to configure at least one authentication domain in the authc of this file. +# An authentication domain is responsible for extracting the user credentials from +# the request and for validating them against an authentication backend like Active Directory for example. +# +# If more than one authentication domain is configured the first one which succeeds wins. +# If all authentication domains fail then the request is unauthenticated. +# In this case an exception is thrown and/or the HTTP status is set to 401. +# +# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect +# the roles from a given backend for the authenticated user. +# +# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both. +# http_enabled: true +# transport_enabled: true +# +# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to +# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated. +# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous" +# and one role named "anonymous_backendrole". +# If you enable anonymous authentication all HTTP authenticators will not challenge. +# +# +# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert" +# first and the challenging one last. +# Because it's not possible to challenge a client with two different authentication methods (for example +# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation +# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request. +# +# Default value of the challenge flag is true. +# +# +# HTTP +# basic (challenging) +# proxy (not challenging, needs xff) +# kerberos (challenging) +# clientcert (not challenging, needs https) +# jwt (not challenging) +# host (not challenging) #DEPRECATED, will be removed in a future version. +# host based authentication is configurable in roles_mapping + +# Authc +# internal +# noop +# ldap + +# Authz +# ldap +# noop + + + +_meta: + type: "config" + config_version: 2 + +config: + dynamic: + # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index + # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default) + # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently + #filtered_alias_mode: warn + #do_not_fail_on_forbidden: false + #kibana: + # Kibana multitenancy + #multitenancy_enabled: true + #server_username: kibanaserver + #index: '.kibana' + http: + anonymous_auth_enabled: false + xff: + enabled: false + internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern + #internalProxies: '.*' # trust all internal proxies, regex pattern + #remoteIpHeader: 'x-forwarded-for' + ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help + ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For + ###### and here https://tools.ietf.org/html/rfc7239 + ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve + authc: + kerberos_auth_domain: + http_enabled: false + transport_enabled: false + order: 6 + http_authenticator: + type: kerberos + challenge: true + config: + # If true a lot of kerberos/security related debugging output will be logged to standard out + krb_debug: false + # If true then the realm will be stripped from the user name + strip_realm_from_principal: true + authentication_backend: + type: noop + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: true + transport_enabled: true + order: 4 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + proxy_auth_domain: + description: "Authenticate via proxy" + http_enabled: false + transport_enabled: false + order: 3 + http_authenticator: + type: proxy + challenge: false + config: + user_header: "x-proxy-user" + roles_header: "x-proxy-roles" + authentication_backend: + type: noop + jwt_auth_domain: + description: "Authenticate via Json Web Token" + http_enabled: false + transport_enabled: false + order: 0 + http_authenticator: + type: jwt + challenge: false + config: + signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key" + jwt_header: "Authorization" + jwt_url_parameter: null + roles_key: null + subject_key: null + authentication_backend: + type: noop + clientcert_auth_domain: + description: "Authenticate via SSL client certificates" + http_enabled: false + transport_enabled: false + order: 2 + http_authenticator: + type: clientcert + config: + username_attribute: cn #optional, if omitted DN becomes username + challenge: false + authentication_backend: + type: noop + ldap: + description: "Authenticate via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + order: 5 + http_authenticator: + type: basic + challenge: false + authentication_backend: + # LDAP authentication backend (authenticate users against a LDAP or Active Directory) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(sAMAccountName={0})' + # Use this attribute from the user as username (if not set then DN is used) + username_attribute: null + authz: + roles_from_myldap: + description: "Authorize via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + rolebase: 'ou=groups,dc=example,dc=com' + # Filter to search for roles (currently in the whole subtree beneath rolebase) + # {0} is substituted with the DN of the user + # {1} is substituted with the username + # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute + rolesearch: '(member={0})' + # Specify the name of the attribute which value should be substituted with {2} above + userroleattribute: null + # Roles as an attribute of the user entry + userrolename: disabled + #userrolename: memberOf + # The attribute in a role entry containing the name of that role, Default is "name". + # Can also be "dn" to use the full DN as rolename. + rolename: cn + # Resolve nested roles transitive (roles which are members of other roles and so on ...) + resolve_nested_roles: true + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(uid={0})' + # Skip users matching a user name, a wildcard or a regex pattern + #skip_users: + # - 'cn=Michael Jackson,ou*people,o=TEST' + # - '/\S*/' + roles_from_another_ldap: + description: "Authorize via another Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + type: ldap + #config goes here ... + # auth_failure_listeners: + # ip_rate_limiting: + # type: ip + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 + # internal_authentication_backend_limiting: + # type: username + # authentication_backend: intern + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/internal_users.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/internal_users.yml new file mode 100644 index 0000000000..f4d31e52c6 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/internal_users.yml @@ -0,0 +1,63 @@ +--- +# This is the internal user database +# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + +_meta: + type: "internalusers" + config_version: 2 + +# Define your internal users here + +## Demo users + +admin: + hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG" + reserved: true + backend_roles: + - "admin" + description: "Demo admin user" + +anomalyadmin: + hash: "$2y$12$TRwAAJgnNo67w3rVUz4FIeLx9Dy/llB79zf9I15CKJ9vkM4ZzAd3." + reserved: false + opendistro_security_roles: + - "anomaly_full_access" + description: "Demo anomaly admin user, using internal role" + +kibanaserver: + hash: "$2a$12$4AcgAt3xwOWadA5s5blL6ev39OXDNhmOesEoo33eZtrq2N0YrU3H." + reserved: true + description: "Demo OpenSearch Dashboards user" + +kibanaro: + hash: "$2a$12$JJSXNfTowz7Uu5ttXfeYpeYE0arACvcwlPBStB1F.MI7f0U9Z4DGC" + reserved: false + backend_roles: + - "kibanauser" + - "readall" + attributes: + attribute1: "value1" + attribute2: "value2" + attribute3: "value3" + description: "Demo OpenSearch Dashboards read only user, using external role mapping" + +logstash: + hash: "$2a$12$u1ShR4l4uBS3Uv59Pa2y5.1uQuZBrZtmNfqB3iM/.jL0XoV9sghS2" + reserved: false + backend_roles: + - "logstash" + description: "Demo logstash user, using external role mapping" + +readall: + hash: "$2a$12$ae4ycwzwvLtZxwZ82RmiEunBbIPiAmGZduBAjKN0TXdwQFtCwARz2" + reserved: false + backend_roles: + - "readall" + description: "Demo readall user, using external role mapping" + +snapshotrestore: + hash: "$2y$12$DpwmetHKwgYnorbgdvORCenv4NAK8cPUg8AI6pxLCuWf/ALc0.v7W" + reserved: false + backend_roles: + - "snapshotrestore" + description: "Demo snapshotrestore user, using external role mapping" diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/nodes_dn.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/nodes_dn.yml new file mode 100644 index 0000000000..7f8304cf0d --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/nodes_dn.yml @@ -0,0 +1,8 @@ +_meta: + type: "nodesdn" + config_version: 2 + +# Define nodesdn mapping name and corresponding values +# cluster1: +# nodes_dn: +# - CN=*.example.com diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example b/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example new file mode 100644 index 0000000000..3b4df645de --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/opensearch.yml.example @@ -0,0 +1,228 @@ +############## OpenSearch Security configuration ############### + +########################################################### +# Add the following settings to your standard opensearch.yml +# alongside with the OpenSearch Security TLS settings. +# Settings must always be the same on all nodes in the cluster. + +############## Common configuration settings ############## + +# Specify a list of DNs which denote the other nodes in the cluster. +# This settings support wildcards and regular expressions +# The list of DNs are also read from security index **in addition** to the yml configuration if +# plugins.security.nodes_dn_dynamic_config_enabled is true. +# NOTE: This setting only has effect if 'plugins.security.cert.intercluster_request_evaluator_class' is not set. +plugins.security.nodes_dn: + - "CN=*.example.com, OU=SSL, O=Test, L=Test, C=DE" + - "CN=node.other.com, OU=SSL, O=Test, L=Test, C=DE" + +# The nodes_dn_dynamic_config_enabled settings is geared towards cross_cluster usecases where there is a need to +# manage the whitelisted nodes_dn without having to restart the nodes everytime a new cross_cluster remote is configured +# Setting nodes_dn_dynamic_config_enabled to true enables **super-admin callable** /_opendistro/_security/api/nodesdn APIs +# which provide means to update/retrieve nodesdn dynamically. +# +# NOTE: The overall whitelisted nodes_dn evaluated comes from both the plugins.security.nodes_dn and the ones stored +# in security index. +# (default: false) +# NOTE2: This setting only has effect if 'plugins.security.cert.intercluster_request_evaluator_class' is not set. +plugins.security.nodes_dn_dynamic_config_enabled: false + +# Defines the DNs (distinguished names) of certificates +# to which admin privileges should be assigned (mandatory) +plugins.security.authcz.admin_dn: + - "CN=kirk,OU=client,O=client,l=tEst, C=De" + +# Define how backend roles should be mapped to Security roles +# MAPPING_ONLY - mappings must be configured explicitely in roles_mapping.yml (default) +# BACKENDROLES_ONLY - backend roles are mapped to Security roles directly. Settings in roles_mapping.yml have no effect. +# BOTH - backend roles are mapped to Security roles mapped directly and via roles_mapping.yml in addition +plugins.security.roles_mapping_resolution: MAPPING_ONLY + +############## REST Management API configuration settings ############## +# Enable or disable role based access to the REST management API +# Default is that no role is allowed to access the REST management API. +#plugins.security.restapi.roles_enabled: ["all_access","xyz_role"] + +# Disable particular endpoints and their HTTP methods for roles. +# By default all endpoints/methods are allowed. +#plugins.security.restapi.endpoints_disabled..: +# Example: +#plugins.security.restapi.endpoints_disabled.all_access.ACTIONGROUPS: ["PUT","POST","DELETE"] +#plugins.security.restapi.endpoints_disabled.xyz_role.LICENSE: ["DELETE"] + +# The following endpoints exist: +# ACTIONGROUPS +# CACHE +# CONFIG +# ROLES +# ROLESMAPPING +# INTERNALUSERS +# SYSTEMINFO +# PERMISSIONSINFO + +############## Auditlog configuration settings ############## +# General settings + +# Enable/disable rest request logging (default: true) +#plugins.security.audit.enable_rest: true +# Enable/disable transport request logging (default: false) +#plugins.security.audit.enable_transport: false +# Enable/disable bulk request logging (default: false) +# If enabled all subrequests in bulk requests will be logged too +#plugins.security.audit.resolve_bulk_requests: false +# Disable some categories +#plugins.security.audit.config.disabled_categories: ["AUTHENTICATED","GRANTED_PRIVILEGES"] +# Disable some requests (wildcard or regex of actions or rest request paths) +#plugins.security.audit.ignore_requests: ["indices:data/read/*","*_bulk"] +# Tune threadpool size, default is 10 +#plugins.security.audit.threadpool.size: 10 +# Tune threadpool max size queue length, default is 100000 +#plugins.security.audit.threadpool.max_queue_len: 100000 + +# Ignore users, e.g. do not log audit requests from that users (default: no ignored users) +#plugins.security.audit.ignore_users: ['kibanaserver','some*user','/also.*regex possible/']" + +# Destination of the auditlog events +plugins.security.audit.type: internal_opensearch +#plugins.security.audit.type: external_opensearch +#plugins.security.audit.type: debug +#plugins.security.audit.type: webhook + +# external_opensearch settings +#plugins.security.audit.config.http_endpoints: ['localhost:9200','localhost:9201','localhost:9202']" +# Auditlog index can be a static one or one with a date pattern (default is 'auditlog6') +#plugins.security.audit.config.index: auditlog6 # make sure you secure this index properly +#plugins.security.audit.config.index: "'auditlog6-'YYYY.MM.dd" #rotates index daily - make sure you secure this index properly +#plugins.security.audit.config.type: auditlog +#plugins.security.audit.config.username: auditloguser +#plugins.security.audit.config.password: auditlogpassword +#plugins.security.audit.config.enable_ssl: false +#plugins.security.audit.config.verify_hostnames: false +#plugins.security.audit.config.enable_ssl_client_auth: false +#plugins.security.audit.config.cert_alias: mycert +#plugins.security.audit.config.pemkey_filepath: key.pem +#plugins.security.audit.config.pemkey_content: <...pem base 64 content> +#plugins.security.audit.config.pemkey_password: secret +#plugins.security.audit.config.pemcert_filepath: cert.pem +#plugins.security.audit.config.pemcert_content: <...pem base 64 content> +#plugins.security.audit.config.pemtrustedcas_filepath: ca.pem +#plugins.security.audit.config.pemtrustedcas_content: <...pem base 64 content> + +# webhook settings +#plugins.security.audit.config.webhook.url: "http://mywebhook/endpoint" +# One of URL_PARAMETER_GET,URL_PARAMETER_POST,TEXT,JSON,SLACK +#plugins.security.audit.config.webhook.format: JSON +#plugins.security.audit.config.webhook.ssl.verify: false +#plugins.security.audit.config.webhook.ssl.pemtrustedcas_filepath: ca.pem +#plugins.security.audit.config.webhook.ssl.pemtrustedcas_content: <...pem base 64 content> + +# log4j settings +#plugins.security.audit.config.log4j.logger_name: auditlogger +#plugins.security.audit.config.log4j.level: INFO + +############## Kerberos configuration settings ############## +# If Kerberos authentication should be used you have to configure: + +# The Path to the krb5.conf file +# Can be absolute or relative to the OpenSearch config directory +#plugins.security.kerberos.krb5_filepath: '/etc/krb5.conf' + +# The Path to the keytab where the acceptor_principal credentials are stored. +# Must be relative to the OpenSearch config directory +#plugins.security.kerberos.acceptor_keytab_filepath: 'eskeytab.tab' + +# Acceptor (Server) Principal name, must be present in acceptor_keytab_path file +#plugins.security.kerberos.acceptor_principal: 'HTTP/localhost' + +############## Advanced configuration settings ############## +# Enable transport layer impersonation +# Allow DNs (distinguished names) to impersonate as other users +#plugins.security.authcz.impersonation_dn: +# "CN=spock,OU=client,O=client,L=Test,C=DE": +# - worf +# "cn=webuser,ou=IT,ou=IT,dc=company,dc=com": +# - user2 +# - user1 + +# Enable rest layer impersonation +# Allow users to impersonate as other users +#plugins.security.authcz.rest_impersonation_user: +# "picard": +# - worf +# "john": +# - steve +# - martin + +# If this is set to true OpenSearch Security will automatically initialize the configuration index +# with the files in the config directory if the index does not exist. +# WARNING: This will use well-known default passwords. +# Use only in a private network/environment. +#plugins.security.allow_default_init_securityindex: false + +# If this is set to true then allow to startup with demo certificates. +# These are certificates issued by floragunn GmbH for demo purposes. +# WARNING: This certificates are well known and therefore unsafe +# Use only in a private network/environment. +#plugins.security.allow_unsafe_democertificates: false + + + +# Password strength rules for password complexity. +# If you want to set up password strength rules for internal users, you can use the below settings for it. +# Password validation rules can be configured through regex. In the below regex example, a user must need +# a password with minimum 8 characters length and must include minimum one uppercase, one lower case, one digit, and one special character.  +# And a custom error message can be configured, in case if a password is not created according to the password strength rule.    +# plugins.security.restapi.password_validation_regex: '(?=.*[A-Z])(?=.*[^a-zA-Z\d])(?=.*[0-9])(?=.*[a-z]).{8,}' +# plugins.security.restapi.password_validation_error_message: "A password must be at least 8 characters long and contain at least one uppercase letter, one lowercase letter, one digit, and one special character." + + +############## Expert settings ############## +# WARNING: Expert settings, do only use if you know what you are doing +# If you set wrong values here this this could be a security risk +# or make OpenSearch Security stop working + +# Name of the index where .opendistro_security stores its configuration. + +#plugins.security.config_index_name: .opendistro_security + +# This defines the OID of server node certificates +#plugins.security.cert.oid: '1.2.3.4.5.5' + +# This specifies the implementation of org.opensearch.security.transport.InterClusterRequestEvaluator +# that is used to determine inter-cluster request. +# Instances of org.opensearch.security.transport.InterClusterRequestEvaluator must implement a single argument +# constructor that takes an org.opensearch.common.settings.Settings +#plugins.security.cert.intercluster_request_evaluator_class: org.opensearch.security.transport.DefaultInterClusterRequestEvaluator + +# By default, normal users can restore snapshots if they have the priviliges 'cluster:admin/snapshot/restore', +# 'indices:admin/create', and 'indices:data/write/index' for the indices to be restored. +# To disable snapshot restore for normal users set 'plugins.security.enable_snapshot_restore_privilege: false'. +# This makes it so that only snapshot restore requests signed by an admin TLS certificate are accepted. +# A snapshot can only be restored when it does not contain global state and does not restore the '.opendistro_security' index +# If 'plugins.security.check_snapshot_restore_write_privileges: false' is set then the additional indices checks are omitted. +#plugins.security.enable_snapshot_restore_privilege: true +#plugins.security.check_snapshot_restore_write_privileges: true + +# Authentication cache timeout in minutes (A value of 0 disables caching, default is 60) +#plugins.security.cache.ttl_minutes: 60 + +# Disable OpenSearch Security +# WARNING: This can expose your configuration (including passwords) to the public. +#plugins.security.disabled: false + + +# Protected indices are even more secure than normal indices. These indices require a role to access like any other index, but they require an additional role +# to be visible, listed in the plugins.security.protected_indices.roles setting. +# Enable protected indices +# plugins.security.protected_indices.enabled: true +# Specify a list of roles a user must be member of to touch any protected index. +# plugins.security.protected_indices.roles: ['all_access'] +# Specify a list of indices to mark as protected. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. +# plugins.security.protected_indices.indices: [] + +# System indices are similar to security index, except the contents are not encrypted. +# Indices configured as system indices can be accessed by only super-admin and no role will provide access to these indices. +# Enable system indices +# plugins.security.system_indices.enabled: true +# Specify a list of indices to mark as system. These indices will only be visible / mutable by members of the above setting, in addition to needing permission to the index via a normal role. +# plugins.security.system_indices.indices: ['.opendistro-alerting-config', '.opendistro-ism-*', '.opendistro-reports-*', '.opensearch-notifications-*', '.opensearch-notebooks', '.opensearch-observability', '.opendistro-asynchronous-search-response*', '.replication-metadata-store'] diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/roles.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/roles.yml new file mode 100644 index 0000000000..1d081a5fd0 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/roles.yml @@ -0,0 +1,292 @@ +_meta: + type: "roles" + config_version: 2 + +# Restrict users so they can only view visualization and dashboard on OpenSearchDashboards +kibana_read_only: + reserved: true + +# The security REST API access role is used to assign specific users access to change the security settings through the REST API. +security_rest_api_access: + reserved: true + +# Allows users to view monitors, destinations and alerts +alerting_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/alerting/alerts/get' + - 'cluster:admin/opendistro/alerting/destination/get' + - 'cluster:admin/opendistro/alerting/monitor/get' + - 'cluster:admin/opendistro/alerting/monitor/search' + - 'cluster:admin/opensearch/alerting/findings/get' + +# Allows users to view and acknowledge alerts +alerting_ack_alerts: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/alerting/alerts/*' + +# Allows users to use all alerting functionality +alerting_full_access: + reserved: true + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opendistro/alerting/*' + - 'cluster:admin/opensearch/alerting/*' + - 'cluster:admin/opensearch/notifications/feature/publish' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + +# Allow users to read Anomaly Detection detectors and results +anomaly_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/ad/detector/info' + - 'cluster:admin/opendistro/ad/detector/search' + - 'cluster:admin/opendistro/ad/detectors/get' + - 'cluster:admin/opendistro/ad/result/search' + - 'cluster:admin/opendistro/ad/tasks/search' + - 'cluster:admin/opendistro/ad/detector/validate' + - 'cluster:admin/opendistro/ad/result/topAnomalies' + +# Allows users to use all Anomaly Detection functionality +anomaly_full_access: + reserved: true + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opendistro/ad/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + - 'indices:admin/aliases/get' + - 'indices:admin/mappings/get' + +# Allows users to read Notebooks +notebooks_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/notebooks/list' + - 'cluster:admin/opendistro/notebooks/get' + +# Allows users to all Notebooks functionality +notebooks_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/notebooks/create' + - 'cluster:admin/opendistro/notebooks/update' + - 'cluster:admin/opendistro/notebooks/delete' + - 'cluster:admin/opendistro/notebooks/get' + - 'cluster:admin/opendistro/notebooks/list' + +# Allows users to read observability objects +observability_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/observability/get' + +# Allows users to all Observability functionality +observability_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/observability/create' + - 'cluster:admin/opensearch/observability/update' + - 'cluster:admin/opensearch/observability/delete' + - 'cluster:admin/opensearch/observability/get' + +# Allows users to read and download Reports +reports_instances_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to read and download Reports and Report-definitions +reports_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/definition/get' + - 'cluster:admin/opendistro/reports/definition/list' + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to all Reports functionality +reports_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/reports/definition/create' + - 'cluster:admin/opendistro/reports/definition/update' + - 'cluster:admin/opendistro/reports/definition/on_demand' + - 'cluster:admin/opendistro/reports/definition/delete' + - 'cluster:admin/opendistro/reports/definition/get' + - 'cluster:admin/opendistro/reports/definition/list' + - 'cluster:admin/opendistro/reports/instance/list' + - 'cluster:admin/opendistro/reports/instance/get' + - 'cluster:admin/opendistro/reports/menu/download' + +# Allows users to use all asynchronous-search functionality +asynchronous_search_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/asynchronous_search/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:data/read/search*' + +# Allows users to read stored asynchronous-search results +asynchronous_search_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opendistro/asynchronous_search/get' + +# Allows user to use all index_management actions - ism policies, rollups, transforms +index_management_full_access: + reserved: true + cluster_permissions: + - "cluster:admin/opendistro/ism/*" + - "cluster:admin/opendistro/rollup/*" + - "cluster:admin/opendistro/transform/*" + - "cluster:admin/opensearch/notifications/feature/publish" + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:admin/opensearch/ism/*' + +# Allows users to use all cross cluster replication functionality at leader cluster +cross_cluster_replication_leader_full_access: + reserved: true + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/read/plugins/replication/changes" + - "indices:data/read/plugins/replication/file_chunk" + +# Allows users to use all cross cluster replication functionality at follower cluster +cross_cluster_replication_follower_full_access: + reserved: true + cluster_permissions: + - "cluster:admin/plugins/replication/autofollow/update" + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - "indices:admin/plugins/replication/index/setup/validate" + - "indices:data/write/plugins/replication/changes" + - "indices:admin/plugins/replication/index/start" + - "indices:admin/plugins/replication/index/pause" + - "indices:admin/plugins/replication/index/resume" + - "indices:admin/plugins/replication/index/stop" + - "indices:admin/plugins/replication/index/update" + - "indices:admin/plugins/replication/index/status_check" + +# Allow users to read ML stats/models/tasks +ml_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/ml/stats/nodes' + - 'cluster:admin/opensearch/ml/models/get' + - 'cluster:admin/opensearch/ml/models/search' + - 'cluster:admin/opensearch/ml/tasks/get' + - 'cluster:admin/opensearch/ml/tasks/search' + +# Allows users to use all ML functionality +ml_full_access: + reserved: true + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opensearch/ml/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + +# Allows users to use all Notifications functionality +notifications_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/notifications/*' + +# Allows users to read Notifications config/channels +notifications_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/notifications/configs/get' + - 'cluster:admin/opensearch/notifications/features' + - 'cluster:admin/opensearch/notifications/channels/get' + +# Allows users to use all snapshot management functionality +snapshot_management_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/snapshot_management/*' + - 'cluster:admin/opensearch/notifications/feature/publish' + - 'cluster:admin/repository/*' + - 'cluster:admin/snapshot/*' + +# Allows users to see snapshots, repositories, and snapshot management policies +snapshot_management_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/snapshot_management/policy/get' + - 'cluster:admin/opensearch/snapshot_management/policy/search' + - 'cluster:admin/opensearch/snapshot_management/policy/explain' + - 'cluster:admin/repository/get' + - 'cluster:admin/snapshot/get' + +# Allows user to use point in time functionality +point_in_time_full_access: + reserved: true + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'manage_point_in_time' + +# Allows users to see security analytics detectors and others +security_analytics_read_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/securityanalytics/alerts/get' + - 'cluster:admin/opensearch/securityanalytics/detector/get' + - 'cluster:admin/opensearch/securityanalytics/detector/search' + - 'cluster:admin/opensearch/securityanalytics/findings/get' + - 'cluster:admin/opensearch/securityanalytics/mapping/get' + - 'cluster:admin/opensearch/securityanalytics/mapping/view/get' + - 'cluster:admin/opensearch/securityanalytics/rule/get' + - 'cluster:admin/opensearch/securityanalytics/rule/search' + +# Allows users to use all security analytics functionality +security_analytics_full_access: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/securityanalytics/alerts/*' + - 'cluster:admin/opensearch/securityanalytics/detector/*' + - 'cluster:admin/opensearch/securityanalytics/findings/*' + - 'cluster:admin/opensearch/securityanalytics/mapping/*' + - 'cluster:admin/opensearch/securityanalytics/rule/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices:admin/mapping/put' + - 'indices:admin/mappings/get' + +# Allows users to view and acknowledge alerts +security_analytics_ack_alerts: + reserved: true + cluster_permissions: + - 'cluster:admin/opensearch/securityanalytics/alerts/*' diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/roles_mapping.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/roles_mapping.yml new file mode 100644 index 0000000000..89f46bf260 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/roles_mapping.yml @@ -0,0 +1,49 @@ +--- +# In this file users, backendroles and hosts can be mapped to Security roles. +# Permissions for OpenSearch roles are configured in roles.yml + +_meta: + type: "rolesmapping" + config_version: 2 + +# Define your roles mapping here + +## Demo roles mapping + +all_access: + reserved: false + backend_roles: + - "admin" + description: "Maps admin to all_access" + +own_index: + reserved: false + users: + - "*" + description: "Allow full access to an index named like the username" + +logstash: + reserved: false + backend_roles: + - "logstash" + +kibana_user: + reserved: false + backend_roles: + - "kibanauser" + description: "Maps kibanauser to kibana_user" + +readall: + reserved: false + backend_roles: + - "readall" + +manage_snapshots: + reserved: false + backend_roles: + - "snapshotrestore" + +kibana_server: + reserved: true + users: + - "kibanaserver" diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/tenants.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/tenants.yml new file mode 100644 index 0000000000..04104dce00 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/tenants.yml @@ -0,0 +1,11 @@ +--- +_meta: + type: "tenants" + config_version: 2 + +# Define your tenants here + +## Demo tenants +admin_tenant: + reserved: false + description: "Demo tenant for admin user" diff --git a/data-node/src/main/resources/opensearch/config/opensearch-security/whitelist.yml b/data-node/src/main/resources/opensearch/config/opensearch-security/whitelist.yml new file mode 100644 index 0000000000..1d54ff841a --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch-security/whitelist.yml @@ -0,0 +1,69 @@ +--- +_meta: + type: "whitelist" + config_version: 2 + +# Description: +# enabled - feature flag. +# if enabled is false, the whitelisting feature is removed. +# This is like removing the check that checks if an API is whitelisted. +# This is equivalent to continuing with the usual access control checks, and removing all the code that implements whitelisting. +# if enabled is true, then all users except SuperAdmin can access only the APIs in requests +# SuperAdmin can access all APIs. +# SuperAdmin is defined by the SuperAdmin certificate, which is configured in the opensearch.yml setting: plugins.security.authcz.admin_dn: +# Refer to the example setting in opensearch.yml.example, and the opendistro documentation to know more about configuring SuperAdmin. +# +# requests - map of whitelisted endpoints, and the whitelisted HTTP requests for those endpoints + +# Examples showing how to configure this yml file (make sure the _meta data from above is also there): +# Example 1: +# To enable whitelisting and whitelist GET /_cluster/settings +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# +# Example 2: +# If you want to whitelist multiple request methods for /_cluster/settings (GET,PUT): +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# - PUT +# +# Example 3: +# If you want to whitelist other APIs as well, for example GET /_cat/nodes, and GET /_cat/shards: +# +#config: +# enabled: true +# requests: +# /_cluster/settings: +# - GET +# - PUT +# /_cat/nodes: +# - GET +# /_cat/shards: +# - GET +# +# Example 4: +# If you want to disable the whitelisting feature, set enabled to false. +# enabled: false +# requests: +# /_cluster/settings: +# - GET +# +#At this point, all APIs become whitelisted because the feature to whitelist is off, so requests is irrelevant. + + +#this name must be config +config: + enabled: false + requests: + /_cluster/settings: + - GET + /_cat/nodes: + - GET diff --git a/data-node/src/main/resources/opensearch/config/opensearch.yml b/data-node/src/main/resources/opensearch/config/opensearch.yml new file mode 100644 index 0000000000..4c3757fbf4 --- /dev/null +++ b/data-node/src/main/resources/opensearch/config/opensearch.yml @@ -0,0 +1,88 @@ +# ======================== OpenSearch Configuration ========================= +# +# NOTE: OpenSearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.opensearch.org +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +#node.name: node-1 +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +#path.data: /path/to/data +# +# Path to log files: +# +#path.logs: /path/to/logs +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +#bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# OpenSearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +#network.host: 192.168.0.1 +# +# Set a custom port for HTTP: +# +#http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +# +# Bootstrap the cluster using an initial set of cluster-manager-eligible nodes: +# +#cluster.initial_cluster_manager_nodes: ["node-1", "node-2"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, consult the gateway module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +#action.destructive_requires_name: true diff --git a/data-node/src/test/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheckTest.java b/data-node/src/test/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheckTest.java index 3c2039293d..4e289bcba2 100644 --- a/data-node/src/test/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheckTest.java +++ b/data-node/src/test/java/org/graylog/datanode/bootstrap/OpensearchBinPreflightCheckTest.java @@ -17,6 +17,7 @@ package org.graylog.datanode.bootstrap; import org.assertj.core.api.Assertions; +import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck; import org.graylog2.bootstrap.preflight.PreflightCheckException; import org.graylog2.shared.utilities.StringUtils; import org.junit.jupiter.api.Test; @@ -28,9 +29,7 @@ import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.util.Collections; -import static org.junit.jupiter.api.Assertions.*; - -class OpensearchBinPreflightCheckTest { +public class OpensearchBinPreflightCheckTest { @TempDir private Path tempDir; diff --git a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java new file mode 100644 index 0000000000..cbaa5deeff --- /dev/null +++ b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/FullDirSyncTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.bootstrap.preflight; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class FullDirSyncTest { + + @TempDir + private Path source; + + @TempDir + private Path target; + + @BeforeEach + void setUp() throws IOException { + + /* + source: + -a.txt + -b.txt + -subdir + -c.txt + + target: + -a.txt + -x.txt + -empty-dir + -subdir + -y.txt + -unused-dir + -z.txt + */ + + Files.createFile(source.resolve("a.txt")); + Files.createFile(source.resolve("b.txt")); + Files.createDirectories(source.resolve("subdir")); + Files.createFile(source.resolve("subdir").resolve("c.txt")); + + Files.createFile(target.resolve("a.txt")); + Files.createFile(target.resolve("x.txt")); + Files.createDirectories(target.resolve("subdir")); + Files.createDirectories(target.resolve("empty-dir")); + Files.createFile(target.resolve("subdir").resolve("y.txt")); + Files.createDirectories(target.resolve("unused-dir")); + Files.createFile(target.resolve("unused-dir").resolve("z.txt")); + } + + @Test + void run() throws IOException { + FullDirSync.run(source, target); + + List afterSyncState = new ArrayList<>(); + + Files.walkFileTree(target, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if(!target.equals(dir)) { + afterSyncState.add(dir); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + afterSyncState.add(file); + return super.visitFile(file, attrs); + } + }); + + Assertions.assertThat(afterSyncState) + .extracting(target::relativize) + .extracting(Path::toString) + .hasSize(4) + .contains("a.txt", "b.txt", "subdir", "subdir/c.txt"); + + } +} diff --git a/data-node/src/test/java/org/graylog/datanode/management/CommandLineProcessTest.java b/data-node/src/test/java/org/graylog/datanode/management/CommandLineProcessTest.java index 9ddac53dc4..5b533340ba 100644 --- a/data-node/src/test/java/org/graylog/datanode/management/CommandLineProcessTest.java +++ b/data-node/src/test/java/org/graylog/datanode/management/CommandLineProcessTest.java @@ -21,16 +21,13 @@ import com.github.rholder.retry.Retryer; import com.github.rholder.retry.RetryerBuilder; import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; -import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.ExecuteException; import org.assertj.core.api.Assertions; -import org.graylog.datanode.ProcessProvidingExecutor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; @@ -62,7 +59,7 @@ class CommandLineProcessTest { } @Test - void testManualStop() throws IOException, ExecutionException, RetryException, URISyntaxException { + void testManualStop() throws IOException, ExecutionException, RetryException { List stdout = new LinkedList<>(); List stdErr = new LinkedList<>(); @@ -93,13 +90,11 @@ class CommandLineProcessTest { LOG.info("On process failed:", e); } }; - final TestExecutor testExecutor = new TestExecutor(); final CommandLineProcess process = new CommandLineProcess( binPath, Collections.emptyList(), listener, - () -> testExecutor, - () -> Map.of("USER", "test", "JAVA_HOME", "/path/to/jre") + new Environment(Map.of("USER", "test", "JAVA_HOME", "/path/to/jre")) ); process.start(); @@ -117,8 +112,6 @@ class CommandLineProcessTest { .hasSize(1) .contains("This message goes to stderr"); - Assertions.assertThat(testExecutor.getEnvironment()) - .doesNotContainKey("JAVA_HOME"); } private void waitTillLogsAreAvailable(List logs, int expectedLinesCount) throws ExecutionException, RetryException { @@ -161,28 +154,11 @@ class CommandLineProcessTest { exitCodeFuture.complete(e.getExitValue()); } }; - final CommandLineProcess process = new CommandLineProcess(binPath, List.of("143"), listener); + final CommandLineProcess process = new CommandLineProcess(binPath, List.of("143"), listener, new Environment(System.getenv())); process.start(); final Integer exitCode = exitCodeFuture.get(10, TimeUnit.SECONDS); Assertions.assertThat(exitCode).isEqualTo(143); } - - /** - * Implementation that allows access to the environment argument. - */ - private static class TestExecutor extends ProcessProvidingExecutor { - private Map environment = Map.of(); - - @Override - protected Process launch(CommandLine command, Map env, File dir) throws IOException { - this.environment = env; - return super.launch(command, env, dir); - } - - public Map getEnvironment() { - return environment; - } - } } diff --git a/data-node/src/test/java/org/graylog/datanode/management/EnvironmentTest.java b/data-node/src/test/java/org/graylog/datanode/management/EnvironmentTest.java new file mode 100644 index 0000000000..11ff2f5829 --- /dev/null +++ b/data-node/src/test/java/org/graylog/datanode/management/EnvironmentTest.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.management; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class EnvironmentTest { + + @Test + void testFiltering() { + final Environment env = new Environment(Map.of("USER", "test", "JAVA_HOME", "/path/to/jre")); + Assertions.assertThat(env.getEnv()) + .doesNotContainKey("JAVA_HOME"); + } +} diff --git a/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeContainerizedBackend.java b/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeContainerizedBackend.java index ed0cbabb9e..087b5ca1b4 100644 --- a/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeContainerizedBackend.java +++ b/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeContainerizedBackend.java @@ -78,12 +78,6 @@ public class DatanodeContainerizedBackend { .withEnv("GRAYLOG_DATANODE_OPENSEARCH_NETWORK_HOST", nodeName) - //datanode_transport_certificate=datanode-transport-certificates.p12 - //datanode_transport_certificate_password=password - - //datanode_http_certificate=datanode-http-certificates.p12 - //datanode_http_certificate_password=password - .withEnv("GRAYLOG_DATANODE_REST_API_USERNAME", "admin") .withEnv("GRAYLOG_DATANODE_REST_API_PASSWORD", "admin") .withEnv("GRAYLOG_DATANODE_PASSWORD_SECRET", "this_is_not_used_but_required")