Remove a data node from the cluster (#17248)

* service, rest & permissions for data node management

* add tests

* add reset data node event

* add states and transitions for node removal
add node removal functionality

* changelog

* add datanode permissions to permission list

* add audit event types to list

* add custom thread factory

* add node removal integration test
refactor rest calls in ITs

* wait for primary shard node availability

* refactor duplicate code

* simplify node retrieval

* add `@OnlyInSecuredNode` annotation and request filter
move delete endpoint to management controller and secure it
adjust IT
change javadoc

* stabilize tests

data-node/src/main/java/org/graylog/datanode/process/ProcessEvent.java

@@ -20,6 +20,8 @@ public enum ProcessEvent {
     PROCESS_STARTED,
     HEALTH_CHECK_OK,
     HEALTH_CHECK_FAILED,
-    PROCESS_STOPPED, // user-triggered action
+    PROCESS_STOPPED,
+    PROCESS_REMOVE,
+    RESET, // user-triggered action
     PROCESS_TERMINATED // failure from outside, not requested
 }

data-node/src/main/java/org/graylog/datanode/process/ProcessState.java

@@ -39,6 +39,16 @@ public enum ProcessState {
      * Failed to reach the opensearch REST API, but the underlying process is still alive
      */
     FAILED,
+
+    /**
+     * Removal of node from Opensearch cluster requested
+     */
+    REMOVING,
+    /**
+     * Removal of node from Opensearch cluster completed
+     */
+    REMOVED,
+    
     /**
      * The OS process is not running anymore on the underlying system, it has been terminated
      */

data-node/src/main/java/org/graylog/datanode/process/ProcessStateMachine.java

@@ -62,6 +62,7 @@ public class ProcessStateMachine {
                 .permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.NOT_RESPONDING)
                 .permit(ProcessEvent.PROCESS_STOPPED, ProcessState.TERMINATED)
                 .permit(ProcessEvent.PROCESS_TERMINATED, ProcessState.TERMINATED)
+                .permit(ProcessEvent.PROCESS_REMOVE, ProcessState.REMOVING)
                 .ignore(ProcessEvent.PROCESS_STARTED);
 
         // if the REST api is not responding, we'll jump to this state and count how many times the failure
@@ -90,6 +91,15 @@ public class ProcessStateMachine {
                 .ignore(ProcessEvent.PROCESS_STOPPED)
                 .ignore(ProcessEvent.PROCESS_TERMINATED); // final state, all following terminate events are ignored
 
+        config.configure(ProcessState.REMOVING)
+                .ignore(ProcessEvent.HEALTH_CHECK_OK)
+                .permit(ProcessEvent.HEALTH_CHECK_FAILED, ProcessState.FAILED)
+                .permit(ProcessEvent.PROCESS_STOPPED, ProcessState.REMOVED);
+
+        config.configure(ProcessState.REMOVED)
+                .permit(ProcessEvent.RESET, ProcessState.WAITING_FOR_CONFIGURATION)
+                .ignore(ProcessEvent.PROCESS_STOPPED);
+
         return new StateMachine<>(ProcessState.WAITING_FOR_CONFIGURATION, config);
     }
 }