opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Process watchdog reimplemented to follow transitions of the process state machine (#15676)

Browse Source 
* Process watchdog reimplemented to follow transitions of the process state machine

* code cleanup

* code cleanup
This commit is contained in:
Tomas Dvorak
2023-06-02 10:52:24 +02:00
committed by GitHub
parent c08b13d392
commit 5651f90aa9
14 changed files with 208 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
data-node/src/test/java/org/graylog/datanode/management/CommandLineProcessTest.java
View File

@@ -23,6 +23,7 @@ import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import org.apache.commons.exec.ExecuteException;
import org.assertj.core.api.Assertions;
import org.graylog.datanode.process.ProcessEvent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;

81
data-node/src/test/java/org/graylog/datanode/management/ProcessWatchdogTest.java
View File

@@ -1,81 +0,0 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.management;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import org.assertj.core.api.Assertions;
import org.graylog.datanode.process.ProcessState;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
class ProcessWatchdogTest {
@Test
void testLifecycle() throws IOException, ExecutionException, RetryException {
final TestableProcess process = new TestableProcess();
final ProcessWatchdog watchdog = new ProcessWatchdog(process, 100);
process.restart();
watchdog.start();
// both process and watchdog are running now. Let's stop the process and see if the watchdog will restart it
process.stop();
// see if the process is starting again
Assertions.assertThat(isInStartingState(process)).isTrue();
// repeat
process.stop();
Assertions.assertThat(isInStartingState(process)).isTrue();
process.stop();
Assertions.assertThat(isInStartingState(process)).isTrue();
// this is the 4th termination, we give up trying
process.stop();
Assertions.assertThat(isWatchdogTerminated(watchdog)).isTrue();
Assertions.assertThat(process.isInState(ProcessState.TERMINATED)).isTrue();
}
private boolean isInStartingState(TestableProcess process) throws ExecutionException, RetryException {
final Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder()
.withWaitStrategy(WaitStrategies.fixedWait(100, TimeUnit.MILLISECONDS))
.withStopStrategy(StopStrategies.stopAfterAttempt(20))
.retryIfResult(Boolean.FALSE::equals)
.build();
return retryer.call(() -> process.isInState(ProcessState.STARTING));
}
private boolean isWatchdogTerminated(ProcessWatchdog watchdog) throws ExecutionException, RetryException {
final Retryer<Boolean> retryer = RetryerBuilder.<Boolean>newBuilder()
.withWaitStrategy(WaitStrategies.fixedWait(100, TimeUnit.MILLISECONDS))
.withStopStrategy(StopStrategies.stopAfterAttempt(5))
.retryIfResult(Boolean.FALSE::equals)
.build();
return retryer.call(watchdog::isStopped);
}
}

61
data-node/src/test/java/org/graylog/datanode/management/ProcessWatchdogTracerTest.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.management;
import org.assertj.core.api.Assertions;
import org.graylog.datanode.process.ProcessEvent;
import org.graylog.datanode.process.ProcessState;
import org.junit.jupiter.api.Test;
class ProcessWatchdogTracerTest {
@Test
void testLifecycle() {
final TestableProcess process = new TestableProcess();
final ProcessWatchdog watchdog = new ProcessWatchdog(process, 3);
process.setStateMachineTracer(watchdog);
process.restart();
// both process and watchdog are running now. Let's stop the process and see if the watchdog will restart it
terminateProcess(process);
// see if the process is starting again
Assertions.assertThat(isInStartingState(process)).isTrue();
// repeat
terminateProcess(process);
Assertions.assertThat(isInStartingState(process)).isTrue();
terminateProcess(process);
Assertions.assertThat(isInStartingState(process)).isTrue();
// this is the 4th termination, we give up trying
terminateProcess(process);
Assertions.assertThat(watchdog.isActive()).isFalse();
Assertions.assertThat(process.isInState(ProcessState.TERMINATED)).isTrue();
}
private void terminateProcess(TestableProcess process) {
process.onEvent(ProcessEvent.PROCESS_TERMINATED);
}
private boolean isInStartingState(TestableProcess process) {
return process.isInState(ProcessState.STARTING);
}
}

20
data-node/src/test/java/org/graylog/datanode/management/TestableProcess.java
View File

@@ -16,8 +16,11 @@
*/
package org.graylog.datanode.management;
import com.github.oxo42.stateless4j.StateMachine;
import org.graylog.datanode.process.ProcessEvent;
import org.graylog.datanode.process.ProcessState;
import org.graylog.datanode.process.ProcessStateMachine;
import org.graylog.datanode.process.StateMachineTracer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,10 +28,10 @@ public class TestableProcess implements ManagableProcess<String> {
private static final Logger LOG = LoggerFactory.getLogger(TestableProcess.class);
private volatile ProcessState state;
private final StateMachine<ProcessState, ProcessEvent> stateMachine;
public TestableProcess() {
this.state = ProcessState.WAITING_FOR_CONFIGURATION;
this.stateMachine = ProcessStateMachine.createNew();
}
@Override
@@ -39,22 +42,27 @@ public class TestableProcess implements ManagableProcess<String> {
@Override
public void restart() {
LOG.debug("Starting process");
this.state = ProcessState.STARTING;
onEvent(ProcessEvent.PROCESS_STARTED);
}
@Override
public void stop() {
LOG.debug("Stopping process process");
this.state = ProcessState.TERMINATED;
onEvent(ProcessEvent.PROCESS_STOPPED);
}
@Override
public void onEvent(ProcessEvent event) {
// ignored
stateMachine.fire(event);
}
@Override
public void setStateMachineTracer(StateMachineTracer stateMachineTracer) {
stateMachine.setTrace(stateMachineTracer);
}
@Override
public boolean isInState(ProcessState state) {
return this.state == state;
return this.stateMachine.isInState(state);
}
}