opensource/graylog2-server
1
0
Fork 0
mirror of https://github.com/Graylog2/graylog2-server.git synced 2026-03-13 09:32:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move lock file check for data dir to later step (#18636)

Browse Source 
* do not write/check for lock file in data dir during preflight test

* remove unnecessary directory read check, handled by validator

* add lockfile check to startup

* remove unused exception

* fix compatibility check url

* add more information and show data node list on restart step

* ignore PROCESS_STOPPED event when in WAITING_FOR_CONFIGURATION state to avoid ugly exception

* fix test and show MigrationDatanodeList
This commit is contained in:
Matthias Oesterheld
2024-03-15 15:20:53 +01:00
committed by GitHub
parent 342c9c3c9b
commit 0a887859ea
10 changed files with 52 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data-node/src/main/java/org/graylog/datanode/Configuration.java
View File

@@ -89,7 +89,7 @@ public class Configuration extends BaseConfiguration {
@Parameter(value = "opensearch_plugins_location", validators = DirectoryReadableValidator.class)
private Path opensearchPluginsDir = Path.of("dist/plugins");
@Parameter(value = "opensearch_data_location", required = true, validators = DirectoryWritableValidator.class)
@Parameter(value = "opensearch_data_location", required = true, validators = DirectoryReadableValidator.class)
private Path opensearchDataLocation = Path.of("datanode/data");
@Parameter(value = "opensearch_logs_location", required = true, validators = DirectoryWritableValidator.class)

6
data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/DatanodeDirectoriesLockfileCheck.java
View File

@@ -17,14 +17,13 @@
package org.graylog.datanode.bootstrap.preflight;
import com.google.common.io.CharStreams;
import jakarta.inject.Inject;
import org.graylog.datanode.configuration.DatanodeConfiguration;
import org.graylog.datanode.configuration.DatanodeDirectories;
import org.graylog2.bootstrap.preflight.PreflightCheck;
import org.graylog2.bootstrap.preflight.PreflightCheckException;
import org.graylog2.plugin.system.NodeId;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
@@ -61,11 +60,10 @@ public class DatanodeDirectoriesLockfileCheck implements PreflightCheck {
@Override
public void runCheck() throws PreflightCheckException {
checkDatanodeLock(directories.getConfigurationTargetDir());
checkDatanodeLock(directories.getDataTargetDir());
checkDatanodeLock(directories.getLogsTargetDir());
}
private void checkDatanodeLock(Path dir) {
public void checkDatanodeLock(Path dir) {
final Path lockfile = dir.resolve(DATANODE_LOCKFILE);
try (FileChannel channel = FileChannel.open(lockfile, StandardOpenOption.READ, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
doCheckLockFile(channel, dir);

29
data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/DatanodeDirectoryException.java
View File

@@ -1,29 +0,0 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.datanode.bootstrap.preflight;
import java.io.IOException;
public class DatanodeDirectoryException extends IOException {
public DatanodeDirectoryException(String message) {
super(message);
}
public DatanodeDirectoryException(String message, IOException cause) {
super(message, cause);
}
}

15
data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java
View File

@@ -24,6 +24,7 @@ import jakarta.inject.Inject;
import jakarta.inject.Provider;
import jakarta.inject.Singleton;
import org.graylog.datanode.Configuration;
import org.graylog.datanode.bootstrap.preflight.DatanodeDirectoriesLockfileCheck;
import org.graylog.datanode.configuration.DatanodeConfiguration;
import org.graylog.datanode.metrics.ConfigureMetricsIndexSettings;
import org.graylog.datanode.process.OpensearchConfiguration;
@@ -58,6 +59,7 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro
private final DataNodeProvisioningService dataNodeProvisioningService;
private final IndexFieldTypesService indexFieldTypesService;
private final ClusterEventBus clusterEventBus;
private final DatanodeDirectoriesLockfileCheck lockfileCheck;
@Inject
@@ -72,13 +74,15 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro
final IndexFieldTypesService indexFieldTypesService,
final ObjectMapper objectMapper,
final ProcessStateMachine processStateMachine,
final ClusterEventBus clusterEventBus) {
final ClusterEventBus clusterEventBus,
final DatanodeDirectoriesLockfileCheck lockfileCheck) {
this.configurationProvider = configurationProvider;
this.eventBus = eventBus;
this.nodeId = nodeId;
this.dataNodeProvisioningService = dataNodeProvisioningService;
this.indexFieldTypesService = indexFieldTypesService;
this.clusterEventBus = clusterEventBus;
this.lockfileCheck = lockfileCheck;
this.process = createOpensearchProcess(datanodeConfiguration, trustManager, configuration, nodeService, objectMapper, processStateMachine);
eventBus.register(this);
}
@@ -142,7 +146,14 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro
final OpensearchConfiguration config = configurationProvider.get();
configure();
if (config.securityConfigured()) {
this.process.start();
try {
lockfileCheck.checkDatanodeLock(config.datanodeDirectories().getDataTargetDir());
this.process.start();
} catch (Exception e) {
LOG.error("Could not start up data node", e);
}
}
}

1
data-node/src/main/java/org/graylog/datanode/process/ProcessStateMachine.java
View File

@@ -48,6 +48,7 @@ public class ProcessStateMachine extends StateMachine<ProcessState, ProcessEvent
.permit(ProcessEvent.PROCESS_PREPARED, ProcessState.PREPARED)
// jump to started only allowed to facilitate startup with insecure config
.permit(ProcessEvent.PROCESS_STARTED, ProcessState.STARTING)
.ignore(ProcessEvent.PROCESS_STOPPED)
.ignore(ProcessEvent.HEALTH_CHECK_FAILED);
config.configure(ProcessState.PREPARED)

33
data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/DatanodeDirectoriesLockfileCheckTest.java
View File

@@ -28,21 +28,22 @@ import java.nio.file.Path;
class DatanodeDirectoriesLockfileCheckTest {
public static final String VALID_NODE_ID = "5ca1ab1e-0000-4000-a000-000000000000";
public static final String OTHER_NODE_ID = "5ca1ab1e-0000-4000-a000-000000000001";
@Test
void testLockCreation(@TempDir Path dataDir,
@TempDir Path logsDir,
@TempDir Path configDir) throws IOException {
final Path dataDirLock = dataDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path logsDirLock = logsDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path configDirLock = configDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck("5ca1ab1e-0000-4000-a000-000000000000", new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck(VALID_NODE_ID, new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
check.runCheck();
Assertions.assertThat(Files.readString(dataDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(logsDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(configDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(logsDirLock)).isEqualTo(VALID_NODE_ID);
Assertions.assertThat(Files.readString(configDirLock)).isEqualTo(VALID_NODE_ID);
}
@@ -51,20 +52,17 @@ class DatanodeDirectoriesLockfileCheckTest {
@TempDir Path logsDir,
@TempDir Path configDir) throws IOException {
final Path dataDirLock = dataDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path logsDirLock = logsDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path configDirLock = configDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
Files.writeString(dataDirLock, "5ca1ab1e-0000-4000-a000-000000000000");
Files.writeString(logsDirLock, "5ca1ab1e-0000-4000-a000-000000000000");
Files.writeString(configDirLock, "5ca1ab1e-0000-4000-a000-000000000000");
Files.writeString(logsDirLock, VALID_NODE_ID);
Files.writeString(configDirLock, VALID_NODE_ID);
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck("5ca1ab1e-0000-4000-a000-000000000000", new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck(VALID_NODE_ID, new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
check.runCheck();
Assertions.assertThat(Files.readString(dataDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(logsDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(configDirLock)).isEqualTo("5ca1ab1e-0000-4000-a000-000000000000");
Assertions.assertThat(Files.readString(logsDirLock)).isEqualTo(VALID_NODE_ID);
Assertions.assertThat(Files.readString(configDirLock)).isEqualTo(VALID_NODE_ID);
}
@Test
@@ -72,18 +70,17 @@ class DatanodeDirectoriesLockfileCheckTest {
@TempDir Path logsDir,
@TempDir Path configDir) throws IOException {
final Path dataDirLock = dataDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path logsDirLock = logsDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
final Path configDirLock = configDir.resolve(DatanodeDirectoriesLockfileCheck.DATANODE_LOCKFILE);
Files.writeString(dataDirLock, "5ca1ab1e-0000-4000-a000-000000000001");
Files.writeString(logsDirLock, "5ca1ab1e-0000-4000-a000-000000000001");
Files.writeString(configDirLock, "5ca1ab1e-0000-4000-a000-000000000001");
Files.writeString(logsDirLock, OTHER_NODE_ID);
Files.writeString(configDirLock, OTHER_NODE_ID);
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck("5ca1ab1e-0000-4000-a000-000000000000", new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
final PreflightCheck check = new DatanodeDirectoriesLockfileCheck(VALID_NODE_ID, new DatanodeDirectories(dataDir, logsDir, null, configDir, null));
Assertions.assertThatThrownBy(check::runCheck)
.isInstanceOf(DatanodeLockFileException.class)
.hasMessageContaining("locked for datanode 5ca1ab1e-0000-4000-a000-000000000001, access with datanode 5ca1ab1e-0000-4000-a000-000000000000 rejected");
}
}

2
graylog2-web-interface/src/components/datanode/hooks/useCompatibilityCheck.ts
View File

@@ -25,7 +25,7 @@ type Options = {
}
const fetchCompatibility = async () => {
const url = 'datanodes/rest/indices-directory/compatibility';
const url = 'datanodes/any/rest/indices-directory/compatibility';
return fetch('GET', qualifyUrl(url));
};

12
graylog2-web-interface/src/components/datanode/migrations/MigrationDatanodeList.tsx
View File

@@ -22,11 +22,15 @@ import { Alert, Table } from 'components/bootstrap';
import { DocumentationLink } from 'components/support';
import useDataNodes from 'components/datanode/hooks/useDataNodes';
type Props = {
showProvisioningState?: boolean
}
const StyledIcon = styled(Icon)`
margin-right: 0.5em;
`;
const MigrationDatanodeList = () => {
const MigrationDatanodeList = ({ showProvisioningState }: Props) => {
const { data: dataNodes, isInitialLoading } = useDataNodes();
if (isInitialLoading) {
@@ -61,7 +65,7 @@ const MigrationDatanodeList = () => {
<tr key={datanode.id}>
<td>{datanode.hostname}</td>
<td>{datanode.transport_address}</td>
<td>{datanode.status}</td>
<td>{showProvisioningState ? datanode.status : datanode.data_node_status}</td>
<td>{datanode.cert_valid_until ? <RelativeTime dateTime={datanode.cert_valid_until} /> : 'No certificate'}</td>
</tr>
))}
@@ -73,4 +77,8 @@ const MigrationDatanodeList = () => {
);
};
MigrationDatanodeList.defaultProps = {
showProvisioningState: true,
};
export default MigrationDatanodeList;

5
graylog2-web-interface/src/components/datanode/migrations/in-place/RestartGraylog.tsx
View File

@@ -19,6 +19,7 @@ import React from 'react';
import type { MigrationStepComponentProps } from 'components/datanode/Types';
import MigrationStepTriggerButtonToolbar from 'components/datanode/migrations/common/MigrationStepTriggerButtonToolbar';
import { Space } from 'preflight/components/common';
import MigrationDatanodeList from 'components/datanode/migrations/MigrationDatanodeList';
const RestartGraylog = ({ currentStep, onTriggerStep }: MigrationStepComponentProps) => (
<>
@@ -26,6 +27,10 @@ const RestartGraylog = ({ currentStep, onTriggerStep }: MigrationStepComponentPr
<p>Please remove the <code>elasticsearch_hosts</code> line from your <code>graylog.conf</code></p>
<p>E.g., <code>elasticsearch_hosts = https://admin:admin@opensearch1:9200,https://admin:admin@opensearch2:9200,https://admin:admin@opensearch3:9200</code></p>
<Space h="md" />
<MigrationDatanodeList showProvisioningState={false} />
<p>Please wait for all data nodes to become &apos;AVAILABLE&apos;. Please check the data node&apos;s log if they do
not become available within 1-2 minutes.
</p>
{/* eslint-disable-next-line react/no-unescaped-entities */}
<p>Once that's done, please restart Graylog to finish the migration.</p>
<MigrationStepTriggerButtonToolbar nextSteps={currentStep.next_steps} onTriggerStep={onTriggerStep} />

4
graylog2-web-interface/src/components/datanode/migrations/in-place/StopMessageProcessing.tsx
View File

@@ -36,6 +36,10 @@ const StopMessageProcessing = ({ currentStep, onTriggerStep }: MigrationStepComp
</Panel.Heading>
<Panel.Body>
<p>Please stop your OpenSearch cluster before proceeding.</p>
<p>If you are migrating existing OpenSearch data by pointing the data node to its data directory, make sure that
the user running the data node (usually graylog-datanode) has permissions to write to the data directory set
in the data node configuration.
</p>
</Panel.Body>
</StyledHelpPanel>
<p />