mirror of
https://github.com/Graylog2/graylog2-server.git
synced 2026-03-13 09:32:21 +08:00
Initial refactoring of data node process/state machine (#19101)
* rename state machine to DataNodeStateMachine to make clear it is not only handling the process anymore * initial refactoring of packages and classes: clearly separate processing and opensearch-related classes * Make ManageableProcess independent from OpensearchStateMachine
This commit is contained in:
Matthias Oesterheld
committed by
GitHub
GitHub
parent
6c4f720107
commit
91ded55629
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import jakarta.validation.constraints.NotNull;
|
||||
import org.apache.commons.exec.CommandLine;
|
||||
import org.apache.commons.exec.DefaultExecutor;
|
||||
import org.apache.commons.exec.ExecuteWatchdog;
|
||||
import org.apache.commons.exec.PumpStreamHandler;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class CommandLineProcess {
|
||||
private final Path executable;
|
||||
private final List<String> arguments;
|
||||
private final ProcessListener listener;
|
||||
private final Environment environment;
|
||||
private final WatchdogWithProcessInfo watchDog;
|
||||
|
||||
public CommandLineProcess(Path executable,
|
||||
List<String> arguments,
|
||||
ProcessListener listener,
|
||||
Environment environment) {
|
||||
this.executable = executable;
|
||||
this.arguments = arguments;
|
||||
this.listener = listener;
|
||||
this.environment = environment;
|
||||
this.watchDog = new WatchdogWithProcessInfo(ExecuteWatchdog.INFINITE_TIMEOUT);
|
||||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(CommandLineProcess.class);
|
||||
|
||||
public void start() {
|
||||
LOG.info("Running process from " + executable.toAbsolutePath());
|
||||
|
||||
CommandLine cmdLine = new CommandLine(executable.toAbsolutePath().toString());
|
||||
arguments.forEach(it -> cmdLine.addArgument(it, true));
|
||||
|
||||
try {
|
||||
createExecutor().execute(cmdLine, environment.getEnv(), listener);
|
||||
listener.onStart();
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private DefaultExecutor createExecutor() {
|
||||
DefaultExecutor executor = new DefaultExecutor();
|
||||
executor.setStreamHandler(new PumpStreamHandler(new LoggingOutputStream(listener::onStdOut), new LoggingOutputStream(listener::onStdErr)));
|
||||
executor.setWatchdog(watchDog);
|
||||
return executor;
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
this.watchDog.destroyProcess();
|
||||
}
|
||||
|
||||
@NotNull
|
||||
public ProcessInformation processInfo() {
|
||||
return watchDog.processInfo();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import org.apache.commons.exec.ExecuteException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* This listener allows to be disabled from outside - if we manually stop the process, we want to stop all the incoming
|
||||
* events as well, otherwise we'll receive a process termination after the stop, which will then confuse our watchdog
|
||||
* and other parts of the datanode.
|
||||
*/
|
||||
public class CommandLineProcessListener implements ProcessListener {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(CommandLineProcessListener.class);
|
||||
|
||||
private final ProcessListener delegate;
|
||||
private boolean listening = true;
|
||||
|
||||
public CommandLineProcessListener(ProcessListener delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
public void stopListening() {
|
||||
this.listening = false;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void onProcessComplete(int exitValue) {
|
||||
if(listening) {
|
||||
delegate.onProcessComplete(exitValue);
|
||||
} else {
|
||||
LOG.info("Ignoring onProcessComplete({}) call, this process is already stopped", exitValue);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onProcessFailed(ExecuteException e) {
|
||||
if(listening) {
|
||||
delegate.onProcessFailed(e);
|
||||
} else {
|
||||
LOG.info("Ignoring onProcessFailed({}) call, this process is already stopped", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStart() {
|
||||
if(listening) {
|
||||
delegate.onStart();
|
||||
} else {
|
||||
LOG.info("Ignoring onStart() call, this process is already stopped");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStdOut(String line) {
|
||||
if(listening) {
|
||||
delegate.onStdOut(line);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStdErr(String line) {
|
||||
if(listening) {
|
||||
delegate.onStdErr(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static java.util.function.Predicate.not;
|
||||
|
||||
public class Environment {
|
||||
|
||||
private static final String JAVA_HOME_ENV = "JAVA_HOME";
|
||||
|
||||
private final Map<String, String> env;
|
||||
private final Map<String, String> additionalVariables = new HashMap<>();
|
||||
|
||||
public Environment(Map<String, String> env) {
|
||||
this.env = env;
|
||||
}
|
||||
|
||||
public Environment put(String key, String value) {
|
||||
this.additionalVariables.put(key, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Map<String, String> getEnv() {
|
||||
Map<String, String> env = new HashMap<>();
|
||||
env.putAll(cleanEnvironment(this.env));
|
||||
env.putAll(additionalVariables);
|
||||
return Collections.unmodifiableMap(env);
|
||||
}
|
||||
|
||||
private Map<String, String> cleanEnvironment(Map<String, String> env) {
|
||||
return env.entrySet().stream()
|
||||
// Remove JAVA_HOME from environment because OpenSearch should use its bundled JVM.
|
||||
.filter(not(entry -> JAVA_HOME_ENV.equals(entry.getKey())))
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
public class FailuresCounter {
|
||||
|
||||
private int counter;
|
||||
private final int initialValue;
|
||||
private final int maxFailuresCount;
|
||||
|
||||
private FailuresCounter(int initialValue, int maxFailuresCount) {
|
||||
this.maxFailuresCount = maxFailuresCount;
|
||||
this.initialValue = initialValue;
|
||||
resetFailuresCounter();
|
||||
}
|
||||
|
||||
public static FailuresCounter oneBased(int maxFailuresCount) {
|
||||
return new FailuresCounter(1, maxFailuresCount);
|
||||
}
|
||||
|
||||
public static FailuresCounter zeroBased(int maxFailuresCount) {
|
||||
return new FailuresCounter(0, maxFailuresCount);
|
||||
}
|
||||
|
||||
public synchronized void increment() {
|
||||
this.counter++;
|
||||
}
|
||||
|
||||
public synchronized boolean failedTooManyTimes() {
|
||||
return this.counter >= maxFailuresCount;
|
||||
}
|
||||
|
||||
public synchronized void resetFailuresCounter() {
|
||||
this.counter = initialValue;
|
||||
}
|
||||
|
||||
public int failuresCount() {
|
||||
return counter;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class LoggingOutputStream extends OutputStream {
|
||||
|
||||
private final StringBuilder builder = new StringBuilder();
|
||||
private final Consumer<String> consumer;
|
||||
|
||||
public LoggingOutputStream(Consumer<String> consumer) {
|
||||
this.consumer = consumer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
if (b == '\n') {
|
||||
consumeLine(builder.toString());
|
||||
builder.setLength(0); // reset the builder
|
||||
} else {
|
||||
builder.append((char) b);
|
||||
}
|
||||
}
|
||||
|
||||
private void consumeLine(String line) {
|
||||
consumer.accept(line);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
super.flush();
|
||||
if(!builder.isEmpty()) {
|
||||
consumeLine(builder.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,9 +17,22 @@
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import com.github.oxo42.stateless4j.delegates.Trace;
|
||||
import org.graylog.datanode.opensearch.statemachine.OpensearchEvent;
|
||||
import org.graylog.datanode.opensearch.statemachine.OpensearchState;
|
||||
import org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTracer;
|
||||
|
||||
public interface ManagableProcess<T, EVENT, STATE> {
|
||||
|
||||
void configure(T configuration);
|
||||
|
||||
void start();
|
||||
|
||||
void stop();
|
||||
|
||||
void onEvent(EVENT event);
|
||||
|
||||
void addStateMachineTracer(Trace<STATE, EVENT> tracer);
|
||||
|
||||
boolean isInState(STATE state);
|
||||
|
||||
/**
|
||||
* The tracer allows to observe triggered event (before) and transitions (after) of the {@link ProcessStateMachine}
|
||||
*/
|
||||
public interface StateMachineTracer extends Trace<ProcessState, ProcessEvent> {
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import org.apache.commons.exec.OS;
|
||||
import org.graylog.datanode.OpensearchDistribution;
|
||||
import org.graylog.datanode.configuration.DatanodeDirectories;
|
||||
import org.graylog.datanode.configuration.S3RepositoryConfiguration;
|
||||
import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration;
|
||||
import org.graylog.datanode.management.Environment;
|
||||
import org.graylog.shaded.opensearch2.org.apache.http.HttpHost;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public record OpensearchConfiguration(
|
||||
OpensearchDistribution opensearchDistribution,
|
||||
DatanodeDirectories datanodeDirectories,
|
||||
String bindAddress,
|
||||
String hostname,
|
||||
int httpPort,
|
||||
int transportPort,
|
||||
String clusterName,
|
||||
String nodeName,
|
||||
List<String> nodeRoles,
|
||||
List<String> discoverySeedHosts,
|
||||
OpensearchSecurityConfiguration opensearchSecurityConfiguration,
|
||||
S3RepositoryConfiguration s3RepositoryConfiguration,
|
||||
|
||||
String nodeSearchCacheSize,
|
||||
Map<String, Object> additionalConfiguration
|
||||
) {
|
||||
public Map<String, Object> asMap() {
|
||||
|
||||
Map<String, Object> config = new LinkedHashMap<>();
|
||||
|
||||
config.put("action.auto_create_index", "false");
|
||||
|
||||
// currently, startup fails on macOS without disabling this filter.
|
||||
// for a description of the filter (although it's for ES), see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html
|
||||
if (OS.isFamilyMac()) {
|
||||
config.put("bootstrap.system_call_filter", "false");
|
||||
}
|
||||
|
||||
if (bindAddress != null && !bindAddress.isBlank()) {
|
||||
config.put("network.host", bindAddress);
|
||||
}
|
||||
config.put("http.port", String.valueOf(httpPort));
|
||||
config.put("transport.port", String.valueOf(transportPort));
|
||||
if (clusterName != null && !clusterName.isBlank()) {
|
||||
config.put("cluster.name", clusterName);
|
||||
}
|
||||
|
||||
config.put("node.name", nodeName);
|
||||
|
||||
if (nodeRoles != null && !nodeRoles.isEmpty()) {
|
||||
config.put("node.roles", toValuesList(nodeRoles));
|
||||
}
|
||||
if (discoverySeedHosts != null && !discoverySeedHosts.isEmpty()) {
|
||||
config.put("discovery.seed_hosts", toValuesList(discoverySeedHosts));
|
||||
}
|
||||
|
||||
config.put("discovery.seed_providers", "file");
|
||||
|
||||
config.put("node.search.cache.size", nodeSearchCacheSize);
|
||||
if (s3RepositoryConfiguration.isRepositoryEnabled()) {
|
||||
config.putAll(s3RepositoryConfiguration.toOpensearchProperties());
|
||||
}
|
||||
|
||||
config.putAll(additionalConfiguration);
|
||||
return config;
|
||||
}
|
||||
|
||||
private String toValuesList(List<String> values) {
|
||||
return String.join(",", values);
|
||||
}
|
||||
|
||||
public Environment getEnv() {
|
||||
final Environment env = new Environment(System.getenv());
|
||||
env.put("OPENSEARCH_JAVA_OPTS", "-Xms%s -Xmx%s".formatted(opensearchSecurityConfiguration.getOpensearchHeap(), opensearchSecurityConfiguration.getOpensearchHeap()));
|
||||
env.put("OPENSEARCH_PATH_CONF", datanodeDirectories.getOpensearchProcessConfigurationDir().toString());
|
||||
return env;
|
||||
}
|
||||
|
||||
public HttpHost getRestBaseUrl() {
|
||||
final boolean sslEnabled = Boolean.parseBoolean(asMap().getOrDefault("plugins.security.ssl.http.enabled", "false").toString());
|
||||
return new HttpHost(hostname(), httpPort(), sslEnabled ? "https" : "http");
|
||||
}
|
||||
|
||||
public HttpHost getClusterBaseUrl() {
|
||||
final boolean sslEnabled = Boolean.parseBoolean(asMap().getOrDefault("plugins.security.ssl.http.enabled", "false").toString());
|
||||
return new HttpHost(hostname(), transportPort(), sslEnabled ? "https" : "http");
|
||||
}
|
||||
|
||||
public boolean securityConfigured() {
|
||||
return opensearchSecurityConfiguration() != null;
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
public enum ProcessEvent {
|
||||
PROCESS_PREPARED,
|
||||
PROCESS_STARTED,
|
||||
HEALTH_CHECK_OK,
|
||||
HEALTH_CHECK_FAILED,
|
||||
PROCESS_STOPPED,
|
||||
PROCESS_REMOVE,
|
||||
RESET, // user-triggered action
|
||||
PROCESS_TERMINATED // failure from outside, not requested
|
||||
}
|
||||
@@ -16,5 +16,10 @@
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
public record OpensearchInfo(String nodeName, ProcessState state, boolean isLeaderNode, String restBaseUrl, ProcessInformation process) {
|
||||
import org.apache.commons.exec.ExecuteResultHandler;
|
||||
|
||||
public interface ProcessListener extends ExecuteResultHandler {
|
||||
void onStart();
|
||||
void onStdOut(String line);
|
||||
void onStdErr(String line);
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import org.graylog2.cluster.nodes.DataNodeStatus;
|
||||
|
||||
public enum ProcessState {
|
||||
/**
|
||||
* Fresh created process, not started yet
|
||||
*/
|
||||
WAITING_FOR_CONFIGURATION(DataNodeStatus.UNCONFIGURED),
|
||||
/**
|
||||
* All configuration files have been written
|
||||
*/
|
||||
PREPARED(DataNodeStatus.PREPARED),
|
||||
/**
|
||||
* The process is running on the underlying OS and has a process ID. It's not responding to the REST API yet
|
||||
*/
|
||||
STARTING(DataNodeStatus.STARTING),
|
||||
/**
|
||||
* Opensearch is now available on the default port with GREEN cluster status, all good
|
||||
*/
|
||||
AVAILABLE(DataNodeStatus.AVAILABLE),
|
||||
/**
|
||||
* There are problems in the REST communication with the opensearch. We'll retry several times before
|
||||
* we go to the FAILED state, giving up.
|
||||
*/
|
||||
NOT_RESPONDING(DataNodeStatus.UNAVAILABLE),
|
||||
|
||||
/**
|
||||
* Failed to reach the opensearch REST API, but the underlying process is still alive
|
||||
*/
|
||||
FAILED(DataNodeStatus.UNAVAILABLE),
|
||||
|
||||
/**
|
||||
* Removal of node from Opensearch cluster requested
|
||||
*/
|
||||
REMOVING(DataNodeStatus.REMOVING),
|
||||
/**
|
||||
* Removal of node from Opensearch cluster completed
|
||||
*/
|
||||
REMOVED(DataNodeStatus.REMOVED),
|
||||
|
||||
/**
|
||||
* The OS process is not running anymore on the underlying system, it has been terminated
|
||||
*/
|
||||
TERMINATED(DataNodeStatus.UNAVAILABLE);
|
||||
|
||||
|
||||
private final DataNodeStatus dataNodeStatus;
|
||||
|
||||
ProcessState(DataNodeStatus dataNodeStatus) {
|
||||
this.dataNodeStatus = dataNodeStatus;
|
||||
}
|
||||
|
||||
public DataNodeStatus getDataNodeStatus() {
|
||||
return dataNodeStatus;
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import com.github.oxo42.stateless4j.StateMachine;
|
||||
import com.github.oxo42.stateless4j.StateMachineConfig;
|
||||
import org.graylog.datanode.management.StateMachineTracerAggregator;
|
||||
|
||||
public class ProcessStateMachine extends StateMachine<ProcessState, ProcessEvent> {
|
||||
/**
|
||||
* How many times can the OS rest api call fail before we switch to the failed state
|
||||
*/
|
||||
public static final int MAX_REST_TEMPORARY_FAILURES = 3;
|
||||
public static final int MAX_REST_STARTUP_FAILURES = 5;
|
||||
public static final int MAX_REBOOT_FAILURES = 3;
|
||||
|
||||
StateMachineTracerAggregator tracerAggregator = new StateMachineTracerAggregator();
|
||||
|
||||
public ProcessStateMachine(ProcessState initialState, StateMachineConfig<ProcessState, ProcessEvent> config) {
|
||||
super(initialState, config);
|
||||
setTrace(tracerAggregator);
|
||||
}
|
||||
|
||||
public static ProcessStateMachine createNew() {
|
||||
|
||||
final FailuresCounter restFailureCounter = FailuresCounter.oneBased(MAX_REST_TEMPORARY_FAILURES);
|
||||
final FailuresCounter startupFailuresCounter = FailuresCounter.oneBased(MAX_REST_STARTUP_FAILURES);
|
||||
final FailuresCounter rebootCounter = FailuresCounter.oneBased(MAX_REBOOT_FAILURES);
|
||||
|
||||
StateMachineConfig<ProcessState, ProcessEvent> config = new StateMachineConfig<>();
|
||||
|
||||
// Freshly created process, it hasn't started yet and doesn't have any pid.
|
||||
config.configure(ProcessState.WAITING_FOR_CONFIGURATION)
|
||||
.permit(ProcessEvent.PROCESS_PREPARED, ProcessState.PREPARED)
|
||||
// jump to started only allowed to facilitate startup with insecure config
|
||||
.permit(ProcessEvent.PROCESS_STARTED, ProcessState.STARTING)
|
||||
.ignore(ProcessEvent.PROCESS_STOPPED)
|
||||
.ignore(ProcessEvent.HEALTH_CHECK_FAILED);
|
||||
|
||||
config.configure(ProcessState.PREPARED)
|
||||
.permit(ProcessEvent.PROCESS_STARTED, ProcessState.STARTING)
|
||||
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
|
||||
.ignore(ProcessEvent.HEALTH_CHECK_FAILED);
|
||||
|
||||
// the process has started already, now we have to wait for a running OS and available REST api
|
||||
// the startupFailuresCounter keeps track of failed REST status calls and allow failures during the
|
||||
// startup period
|
||||
config.configure(ProcessState.STARTING)
|
||||
.permitDynamic(ProcessEvent.HEALTH_CHECK_FAILED,
|
||||
() -> startupFailuresCounter.failedTooManyTimes() ? ProcessState.FAILED : ProcessState.STARTING,
|
||||
startupFailuresCounter::increment)
|
||||
.permit(ProcessEvent.HEALTH_CHECK_OK, ProcessState.AVAILABLE)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED);
|
||||
|
||||
// the process is running and responding to the REST status, it's available for any usage
|
||||
config.configure(ProcessState.AVAILABLE)
|
||||
.onEntry(restFailureCounter::resetFailuresCounter)
|
||||
.onEntry(rebootCounter::resetFailuresCounter)
|
||||
.permitReentry(ProcessEvent.HEALTH_CHECK_OK)
|
||||
.permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.NOT_RESPONDING)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_REMOVE, ProcessState.REMOVING)
|
||||
.ignore(ProcessEvent.PROCESS_STARTED);
|
||||
|
||||
// if the REST api is not responding, we'll jump to this state and count how many times the failure
|
||||
// occurs. If it fails ttoo many times, we'll mark the process as FAILED
|
||||
config.configure(ProcessState.NOT_RESPONDING)
|
||||
.permitDynamic(ProcessEvent.HEALTH_CHECK_FAILED,
|
||||
() -> restFailureCounter.failedTooManyTimes() ? ProcessState.FAILED : ProcessState.NOT_RESPONDING,
|
||||
restFailureCounter::increment
|
||||
)
|
||||
.permit(ProcessEvent.HEALTH_CHECK_OK, ProcessState.AVAILABLE)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED);
|
||||
|
||||
// failed and we see the process as not recoverable.
|
||||
// TODO: what to do if the process fails? Reboot?
|
||||
config.configure(ProcessState.FAILED)
|
||||
.ignore(ProcessEvent.HEALTH_CHECK_FAILED)
|
||||
.permit(ProcessEvent.HEALTH_CHECK_OK, ProcessState.AVAILABLE)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED);
|
||||
|
||||
// final state, the process is not alive anymore, terminated on the operating system level
|
||||
config.configure(ProcessState.TERMINATED)
|
||||
.permit(ProcessEvent.PROCESS_STARTED, ProcessState.STARTING, rebootCounter::increment)
|
||||
.ignore(ProcessEvent.HEALTH_CHECK_FAILED)
|
||||
.ignore(ProcessEvent.PROCESS_STOPPED)
|
||||
.ignore(ProcessEvent.PROCESS_TERMINATED); // final state, all following terminate events are ignored
|
||||
|
||||
config.configure(ProcessState.REMOVING)
|
||||
.ignore(ProcessEvent.HEALTH_CHECK_OK)
|
||||
.permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.FAILED)
|
||||
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.REMOVED);
|
||||
|
||||
config.configure(ProcessState.REMOVED)
|
||||
.permit(ProcessEvent.RESET, ProcessState.WAITING_FOR_CONFIGURATION)
|
||||
.ignore(ProcessEvent.PROCESS_STOPPED);
|
||||
|
||||
return new ProcessStateMachine(ProcessState.WAITING_FOR_CONFIGURATION, config);
|
||||
}
|
||||
|
||||
public StateMachineTracerAggregator getTracerAggregator() {
|
||||
return tracerAggregator;
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.process;
|
||||
|
||||
import jakarta.inject.Provider;
|
||||
import jakarta.inject.Inject;
|
||||
|
||||
public class ProcessStateMachineProvider implements Provider<ProcessStateMachine> {
|
||||
private final ProcessStateMachine processStateMachine;
|
||||
|
||||
@Inject
|
||||
public ProcessStateMachineProvider() {
|
||||
this.processStateMachine = ProcessStateMachine.createNew();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProcessStateMachine get() {
|
||||
return processStateMachine;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user