opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove a data node from the cluster (#17248)

Browse Source 
* service, rest & permissions for data node management

* add tests

* add reset data node event

* add states and transitions for node removal
add node removal functionality

* changelog

* add datanode permissions to permission list

* add audit event types to list

* add custom thread factory

* add node removal integration test
refactor rest calls in ITs

* wait for primary shard node availability

* refactor duplicate code

* simplify node retrieval

* add `@OnlyInSecuredNode` annotation and request filter
move delete endpoint to management controller and secure it
adjust IT
change javadoc

* stabilize tests
This commit is contained in:
Matthias Oesterheld
2023-11-20 13:33:26 +01:00
committed by GitHub
parent 51b4f4c226
commit c6101f395c
44 changed files with 1519 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data-node/src/main/java/org/graylog/datanode/initializers/JerseyService.java
View File

@@ -42,6 +42,7 @@ import org.graylog.datanode.configuration.variants.KeystoreInformation;
import org.graylog.datanode.configuration.variants.OpensearchSecurityConfiguration;
import org.graylog.datanode.management.OpensearchConfigurationChangeEvent;
import org.graylog.datanode.process.OpensearchConfiguration;
import org.graylog.datanode.rest.config.SecuredNodeAnnotationFilter;
import org.graylog.security.certutil.CertConstants;
import org.graylog2.bootstrap.preflight.web.BasicAuthFilter;
import org.graylog2.configuration.TLSProtocolsConfiguration;
@@ -231,6 +232,7 @@ public class JerseyService extends AbstractIdleService {
if(isSecuredInstance) {
resourceConfig.register(new BasicAuthFilter(configuration.getRootUsername(), configuration.getRootPasswordSha2(), "Datanode"));
}
resourceConfig.register(new SecuredNodeAnnotationFilter(configuration.isInsecureStartup()));
final HttpServer httpServer = GrizzlyHttpServerFactory.createHttpServer(
listenUri,

4
data-node/src/main/java/org/graylog/datanode/management/OpensearchProcess.java
View File

@@ -37,4 +37,8 @@ public interface OpensearchProcess extends ManagableProcess<OpensearchConfigurat
URI getOpensearchBaseUrl();
String getOpensearchClusterUrl();
void onRemove();
void onReset();
}

13
data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessImpl.java
View File

@@ -203,6 +203,19 @@ class OpensearchProcessImpl implements OpensearchProcess, ProcessListener {
}
}
@Override
public void onRemove() {
LOG.info("Starting removal of OpenSearch node");
if (this.commandLineProcess != null) {
onEvent(ProcessEvent.PROCESS_REMOVE);
}
}
@Override
public void onReset() {
onEvent(ProcessEvent.RESET);
}
@Override
public void onStart() {
onEvent(ProcessEvent.PROCESS_STARTED);

25
data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java
View File

@@ -24,6 +24,9 @@ import org.graylog.datanode.configuration.DatanodeConfiguration;
import org.graylog.datanode.process.OpensearchConfiguration;
import org.graylog2.cluster.NodeService;
import org.graylog2.cluster.preflight.DataNodeProvisioningStateChangeEvent;
import org.graylog2.datanode.DataNodeLifecycleEvent;
import org.graylog2.events.ClusterEventBus;
import org.graylog2.plugin.system.NodeId;
import org.graylog2.security.CustomCAX509TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +35,8 @@ import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
import static com.google.common.base.Preconditions.checkNotNull;
@Singleton
public class OpensearchProcessService extends AbstractIdleService implements Provider<OpensearchProcess> {
@@ -41,18 +46,23 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro
private final OpensearchProcess process;
private final Provider<OpensearchConfiguration> configurationProvider;
private final EventBus eventBus;
private final NodeId nodeId;
@Inject
public OpensearchProcessService(final DatanodeConfiguration datanodeConfiguration,
final Provider<OpensearchConfiguration> configurationProvider,
final EventBus eventBus,
final ClusterEventBus clusterEventBus,
final CustomCAX509TrustManager trustManager,
final NodeService nodeService,
final Configuration configuration) {
final Configuration configuration,
final NodeId nodeId) {
this.configurationProvider = configurationProvider;
this.eventBus = eventBus;
this.nodeId = nodeId;
this.process = createOpensearchProcess(datanodeConfiguration, trustManager, configuration, nodeService);
eventBus.register(this);
checkNotNull(clusterEventBus).registerClusterEventSubscriber(this);
}
private OpensearchProcess createOpensearchProcess(final DatanodeConfiguration datanodeConfiguration, final CustomCAX509TrustManager trustManager, final Configuration configuration, final NodeService nodeService) {
@@ -60,16 +70,29 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro
final ProcessWatchdog watchdog = new ProcessWatchdog(process, WATCHDOG_RESTART_ATTEMPTS);
process.addStateMachineTracer(watchdog);
process.addStateMachineTracer(new StateMachineTransitionLogger());
process.addStateMachineTracer(new OpensearchRemovalTracer(process, configuration.getDatanodeNodeName()));
return process;
}
@Subscribe
@SuppressWarnings("unused")
public void handlePreflightConfigEvent(DataNodeProvisioningStateChangeEvent event) {
switch (event.state()) {
case STORED -> startWithConfig();
}
}
@Subscribe
@SuppressWarnings("unused")
public void handleNodeLifecycleEvent(DataNodeLifecycleEvent event) {
if (nodeId.getNodeId().equals(event.nodeId())) {
switch (event.trigger()) {
case REMOVE -> process.onRemove();
case RESET -> process.onReset();
}
}
}
@Override
protected void startUp() {
startWithConfig();

142
data-node/src/main/java/org/graylog/datanode/management/OpensearchRemovalTracer.java Normal file
View File

@@ -0,0 +1,142 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.management;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.graylog.datanode.process.ProcessEvent;
import org.graylog.datanode.process.ProcessState;
import org.graylog.datanode.process.StateMachineTracer;
import org.graylog.shaded.opensearch2.org.opensearch.OpenSearchStatusException;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.graylog.shaded.opensearch2.org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
import org.graylog.shaded.opensearch2.org.opensearch.client.ClusterClient;
import org.graylog.shaded.opensearch2.org.opensearch.client.RequestOptions;
import org.graylog.shaded.opensearch2.org.opensearch.client.RestHighLevelClient;
import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* This tracer triggers functionality for node removal.
* It triggers removal on PROCESS_REMOVE and will reset the allocation exclude setting in OS on successful start of OS,
* which cannot be done after the removal due to the unavailability of the OS REST api.
*/
public class OpensearchRemovalTracer implements StateMachineTracer {
private static final Logger LOG = LoggerFactory.getLogger(OpensearchRemovalTracer.class);
static final String CLUSTER_ROUTING_ALLOCATION_EXCLUDE_SETTING = "cluster.routing.allocation.exclude._name";
private final OpensearchProcess process;
private final String nodeName;
boolean allocationExcludeChecked = false;
ScheduledExecutorService executorService;
public OpensearchRemovalTracer(OpensearchProcess process, String nodeName) {
this.process = process;
this.nodeName = nodeName;
}
@Override
public void trigger(ProcessEvent trigger) {
LOG.debug("Removal tracer trigger: {}", trigger);
}
@Override
public void transition(ProcessEvent trigger, ProcessState source, ProcessState destination) {
if (destination == ProcessState.AVAILABLE && !allocationExcludeChecked) {
checkAllocationEnabledStatus();
} else if (trigger == ProcessEvent.PROCESS_REMOVE) {
triggerRemoval();
}
}
private void checkAllocationEnabledStatus() {
if (process.restClient().isPresent()) {
ClusterClient clusterClient = process.restClient().get().cluster();
try {
final ClusterGetSettingsResponse settings =
clusterClient.getSettings(new ClusterGetSettingsRequest(), RequestOptions.DEFAULT);
final String setting = settings.getSetting(CLUSTER_ROUTING_ALLOCATION_EXCLUDE_SETTING);
if (nodeName.equals(setting)) {
ClusterUpdateSettingsRequest updateSettings = new ClusterUpdateSettingsRequest();
updateSettings.transientSettings(Settings.builder()
.putNull(CLUSTER_ROUTING_ALLOCATION_EXCLUDE_SETTING)
.build());
clusterClient.putSettings(updateSettings, RequestOptions.DEFAULT);
}
allocationExcludeChecked = true;
} catch (IOException e) {
LOG.error("Error getting cluster settings from OpenSearch");
}
}
}
private void triggerRemoval() {
process.restClient().ifPresent(client -> {
final ClusterClient clusterClient = client.cluster();
ClusterUpdateSettingsRequest settings = new ClusterUpdateSettingsRequest();
settings.transientSettings(Settings.builder()
.put(CLUSTER_ROUTING_ALLOCATION_EXCLUDE_SETTING, nodeName)
.build());
try {
final ClusterUpdateSettingsResponse response =
clusterClient.putSettings(settings, RequestOptions.DEFAULT);
if (response.isAcknowledged()) {
allocationExcludeChecked = false; // reset to rejoin cluster in case of failure
executorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("datanode-removal").build());
executorService.scheduleAtFixedRate(this::checkRemovalStatus, 10, 10, TimeUnit.SECONDS);
} else {
process.onEvent(ProcessEvent.HEALTH_CHECK_FAILED);
}
} catch (IOException e) {
LOG.error("Failed to exclude node from cluster allocation", e);
process.onEvent(ProcessEvent.HEALTH_CHECK_FAILED);
}
});
}
void checkRemovalStatus() {
final Optional<RestHighLevelClient> restClient = process.restClient();
if (restClient.isPresent()) {
try {
final ClusterClient clusterClient = restClient.get().cluster();
final ClusterHealthResponse health = clusterClient
.health(new ClusterHealthRequest(), RequestOptions.DEFAULT);
if (health.getRelocatingShards() == 0) {
process.stop();
executorService.shutdown();
// TODO: set the state for this datanode to removed in nodes
// and include state in OpensearchProcessImpl.writeSeedHostsList
}
} catch (IOException | OpenSearchStatusException e) {
process.onEvent(ProcessEvent.HEALTH_CHECK_FAILED);
}
}
}
}

3
data-node/src/main/java/org/graylog/datanode/periodicals/OpensearchNodeHeartbeat.java
View File

@@ -47,7 +47,8 @@ public class OpensearchNodeHeartbeat extends Periodical {
@Override
// This method is "synchronized" because we are also calling it directly in AutomaticLeaderElectionService
public synchronized void doRun() {
if (!process.isInState(ProcessState.TERMINATED) && !process.isInState(ProcessState.WAITING_FOR_CONFIGURATION)) {
if (!process.isInState(ProcessState.TERMINATED) && !process.isInState(ProcessState.WAITING_FOR_CONFIGURATION)
&& !process.isInState(ProcessState.REMOVED)) {
final Optional<RestHighLevelClient> restClient = process.restClient();
if(restClient.isPresent()) {

4
data-node/src/main/java/org/graylog/datanode/process/ProcessEvent.java
View File

@@ -20,6 +20,8 @@ public enum ProcessEvent {
PROCESS_STARTED,
HEALTH_CHECK_OK,
HEALTH_CHECK_FAILED,
PROCESS_STOPPED, // user-triggered action
PROCESS_STOPPED,
PROCESS_REMOVE,
RESET, // user-triggered action
PROCESS_TERMINATED // failure from outside, not requested
}

10
data-node/src/main/java/org/graylog/datanode/process/ProcessState.java
View File

@@ -39,6 +39,16 @@ public enum ProcessState {
* Failed to reach the opensearch REST API, but the underlying process is still alive
*/
FAILED,
/**
* Removal of node from Opensearch cluster requested
*/
REMOVING,
/**
* Removal of node from Opensearch cluster completed
*/
REMOVED,
/**
* The OS process is not running anymore on the underlying system, it has been terminated
*/

10
data-node/src/main/java/org/graylog/datanode/process/ProcessStateMachine.java
View File

@@ -62,6 +62,7 @@ public class ProcessStateMachine {
.permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.NOT_RESPONDING)
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
.permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED)
.permit(ProcessEvent.PROCESS_REMOVE, ProcessState.REMOVING)
.ignore(ProcessEvent.PROCESS_STARTED);
// if the REST api is not responding, we'll jump to this state and count how many times the failure
@@ -90,6 +91,15 @@ public class ProcessStateMachine {
.ignore(ProcessEvent.PROCESS_STOPPED)
.ignore(ProcessEvent.PROCESS_TERMINATED); // final state, all following terminate events are ignored
config.configure(ProcessState.REMOVING)
.ignore(ProcessEvent.HEALTH_CHECK_OK)
.permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.FAILED)
.permit(ProcessEvent.PROCESS_STOPPED, ProcessState.REMOVED);
config.configure(ProcessState.REMOVED)
.permit(ProcessEvent.RESET, ProcessState.WAITING_FOR_CONFIGURATION)
.ignore(ProcessEvent.PROCESS_STOPPED);
return new StateMachine<>(ProcessState.WAITING_FOR_CONFIGURATION, config);
}
}

45
data-node/src/main/java/org/graylog/datanode/rest/ManagementController.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.rest;
import org.graylog.datanode.management.OpensearchProcess;
import org.graylog.datanode.rest.config.OnlyInSecuredNode;
import javax.inject.Inject;
import javax.ws.rs.DELETE;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
@Path("/management")
@Produces(MediaType.APPLICATION_JSON)
public class ManagementController {
private final OpensearchProcess openSearch;
@Inject
public ManagementController(OpensearchProcess openSearch) {
this.openSearch = openSearch;
}
@DELETE
@OnlyInSecuredNode
public void remove() {
openSearch.onRemove();
}
}

1
data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java
View File

@@ -23,5 +23,6 @@ public class RestBindings extends Graylog2Module {
protected void configure() {
addSystemRestResource(StatusController.class);
addSystemRestResource(LogsController.class);
addSystemRestResource(ManagementController.class);
}
}

1
data-node/src/main/java/org/graylog/datanode/rest/StatusController.java
View File

@@ -48,4 +48,5 @@ public class StatusController {
new StatusResponse(datanodeConfiguration.opensearchDistributionProvider().get().version(), openSearch.processInfo())
);
}
}

27
data-node/src/main/java/org/graylog/datanode/rest/config/OnlyInSecuredNode.java Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.rest.config;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface OnlyInSecuredNode {
}

60
data-node/src/main/java/org/graylog/datanode/rest/config/SecuredNodeAnnotationFilter.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.rest.config;
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.model.ResourceMethod;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.util.Objects;
public class SecuredNodeAnnotationFilter implements ContainerRequestFilter {
private final boolean isInsecureNode;
public SecuredNodeAnnotationFilter(boolean isInsecureNode) {
this.isInsecureNode = isInsecureNode;
}
@Override
public void filter(ContainerRequestContext requestContext) throws IOException {
if (requestContext instanceof ContainerRequest request) {
final ResourceMethod method = request.getUriInfo().getMatchedResourceMethod();
final OnlyInSecuredNode annotation = method.getInvocable()
.getHandlingMethod().getAnnotation(OnlyInSecuredNode.class);
if (Objects.nonNull(annotation) && isInsecureNode) {
requestContext.abortWith(Response.status(Response.Status.UNAUTHORIZED)
.entity("This resource can only be accessed in secured data nodes.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
} else {
requestContext.abortWith(Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity("Server is no Jetty Server.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
}
}