mirror of
https://github.com/Graylog2/graylog2-server.git
synced 2026-03-13 09:32:21 +08:00
Parser for opensearch/elasticsearch data directories (#17487)
* Parser for opensearch/elasticsearch data directories * add opensearch 1.3 and elasticsearch 6.8.0 to indexer parser * fixed paths * code cleanup * remove explicit dependency version of jackson smile * Preflight check for index dir compatibility * indexer dir parser fixed for empty dirs * added changelog * rest controller for indices directory parser * require auth for indices compatibility rest api * catch incompatibility exceptions of indices dir parsing * code cleanup * better dir detection handling --------- Co-authored-by: Matthias Oesterheld <33032967+moesterheld@users.noreply.github.com>
This commit is contained in:
Tomas Dvorak
@@ -19,6 +19,10 @@ package org.graylog.datanode.bindings;
|
||||
import com.google.inject.AbstractModule;
|
||||
import org.graylog.datanode.configuration.DatanodeConfiguration;
|
||||
import org.graylog.datanode.configuration.DatanodeConfigurationProvider;
|
||||
import org.graylog.datanode.filesystem.index.indexreader.ShardStatsParser;
|
||||
import org.graylog.datanode.filesystem.index.indexreader.ShardStatsParserImpl;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFileParser;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFileParserImpl;
|
||||
import org.graylog2.plugin.system.FilePersistedNodeIdProvider;
|
||||
import org.graylog2.plugin.system.NodeId;
|
||||
import org.graylog.datanode.configuration.OpensearchDistributionProvider;
|
||||
@@ -30,5 +34,7 @@ public class DatanodeConfigurationBindings extends AbstractModule {
|
||||
bind(NodeId.class).toProvider(FilePersistedNodeIdProvider.class).asEagerSingleton();
|
||||
bind(DatanodeConfiguration.class).toProvider(DatanodeConfigurationProvider.class);
|
||||
bind(OpensearchDistribution.class).toProvider(OpensearchDistributionProvider.class);
|
||||
bind(StateFileParser.class).to(StateFileParserImpl.class);
|
||||
bind(ShardStatsParser.class).to(ShardStatsParserImpl.class);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import org.graylog.datanode.bootstrap.preflight.DatanodeDirectoriesLockfileCheck
|
||||
import org.graylog.datanode.bootstrap.preflight.OpenSearchPreconditionsCheck;
|
||||
import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck;
|
||||
import org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync;
|
||||
import org.graylog.datanode.bootstrap.preflight.OpensearchDataDirCompatibilityCheck;
|
||||
import org.graylog2.bootstrap.preflight.PreflightCheck;
|
||||
|
||||
public class PreflightChecksBindings extends AbstractModule {
|
||||
@@ -33,6 +34,7 @@ public class PreflightChecksBindings extends AbstractModule {
|
||||
addPreflightCheck(OpensearchBinPreflightCheck.class);
|
||||
addPreflightCheck(DatanodeDirectoriesLockfileCheck.class);
|
||||
addPreflightCheck(OpenSearchPreconditionsCheck.class);
|
||||
addPreflightCheck(OpensearchDataDirCompatibilityCheck.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.bootstrap.preflight;
|
||||
|
||||
import org.graylog.datanode.configuration.DatanodeConfiguration;
|
||||
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
|
||||
import org.graylog.datanode.filesystem.index.IndicesDirectoryParser;
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.NodeInformation;
|
||||
import org.graylog.shaded.opensearch2.org.opensearch.Version;
|
||||
import org.graylog2.bootstrap.preflight.PreflightCheck;
|
||||
import org.graylog2.bootstrap.preflight.PreflightCheckException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class OpensearchDataDirCompatibilityCheck implements PreflightCheck {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(OpensearchDataDirCompatibilityCheck.class);
|
||||
|
||||
private final boolean isFreshInstallation;
|
||||
private final DatanodeConfiguration datanodeConfiguration;
|
||||
private final IndicesDirectoryParser indicesDirectoryParser;
|
||||
|
||||
@Inject
|
||||
public OpensearchDataDirCompatibilityCheck(@Named("isFreshInstallation") boolean isFreshInstallation, DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) {
|
||||
this.isFreshInstallation = isFreshInstallation;
|
||||
this.datanodeConfiguration = datanodeConfiguration;
|
||||
this.indicesDirectoryParser = indicesDirectoryParser;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void runCheck() throws PreflightCheckException {
|
||||
|
||||
if (!isFreshInstallation) {
|
||||
return;
|
||||
}
|
||||
|
||||
final Path opensearchDataDir = datanodeConfiguration.datanodeDirectories().getDataTargetDir();
|
||||
final String opensearchVersion = datanodeConfiguration.opensearchDistributionProvider().get().version();
|
||||
|
||||
try {
|
||||
final IndexerDirectoryInformation info = indicesDirectoryParser.parse(opensearchDataDir);
|
||||
checkCompatibility(opensearchVersion, info);
|
||||
final int indicesCount = info.nodes().stream().mapToInt(n -> n.indices().size()).sum();
|
||||
LOG.info("Found {} indices and all of them are valid with current opensearch version {}", indicesCount, opensearchVersion);
|
||||
} catch (IncompatibleIndexVersionException e) {
|
||||
throw new PreflightCheckException("Index directory is not compatible with current version " + opensearchVersion + " of Opensearch, terminating.", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void checkCompatibility(String opensearchVersion, IndexerDirectoryInformation info) {
|
||||
final Version currentVersion = Version.fromString(opensearchVersion);
|
||||
for (NodeInformation node : info.nodes()) {
|
||||
final Version nodeVersion = Version.fromString(node.nodeVersion());
|
||||
if (node.nodeVersion() != null && !currentVersion.isCompatible(nodeVersion)) {
|
||||
final String error = String.format(Locale.ROOT, "Current version %s of Opensearch is not compatible with index version %s", currentVersion, nodeVersion);
|
||||
throw new IncompatibleIndexVersionException(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index;
|
||||
|
||||
public class IncompatibleIndexVersionException extends RuntimeException {
|
||||
public IncompatibleIndexVersionException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
public IncompatibleIndexVersionException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index;
|
||||
|
||||
public class IndexerInformationParserException extends RuntimeException {
|
||||
public IndexerInformationParserException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public IndexerInformationParserException(String message, Exception cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index;
|
||||
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.NodeInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.ShardInformation;
|
||||
import org.graylog.datanode.filesystem.index.indexreader.ShardStats;
|
||||
import org.graylog.datanode.filesystem.index.indexreader.ShardStatsParser;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFile;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFileParser;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Singleton
|
||||
public class IndicesDirectoryParser {
|
||||
|
||||
public static final String STATE_DIR_NAME = "_state";
|
||||
public static final String STATE_FILE_EXTENSION = ".st";
|
||||
|
||||
private final StateFileParser stateFileParser;
|
||||
private final ShardStatsParser shardReader;
|
||||
|
||||
@Inject
|
||||
public IndicesDirectoryParser(StateFileParser stateFileParser, ShardStatsParser shardReader) {
|
||||
this.stateFileParser = stateFileParser;
|
||||
this.shardReader = shardReader;
|
||||
}
|
||||
|
||||
public IndexerDirectoryInformation parse(Path path) {
|
||||
if (!Files.exists(path)) {
|
||||
throw new IndexerInformationParserException("Path " + path + " does not exist.");
|
||||
}
|
||||
|
||||
if (!Files.isDirectory(path)) {
|
||||
throw new IndexerInformationParserException("Path " + path + " is not a directory");
|
||||
}
|
||||
|
||||
if (!Files.isReadable(path)) {
|
||||
throw new IndexerInformationParserException("Path " + path + " is not readable");
|
||||
}
|
||||
|
||||
final Path nodesPath = path.resolve("nodes");
|
||||
|
||||
if(!Files.exists(nodesPath)) {
|
||||
return IndexerDirectoryInformation.empty(path);
|
||||
}
|
||||
|
||||
try (final Stream<Path> nodes = Files.list(nodesPath)) {
|
||||
final List<NodeInformation> nodeInformation = nodes.filter(Files::isDirectory)
|
||||
.filter(p -> p.getFileName().toString().matches("\\d+"))
|
||||
.map(this::parseNode)
|
||||
.toList();
|
||||
return new IndexerDirectoryInformation(path, nodeInformation);
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to list nodes", e);
|
||||
}
|
||||
}
|
||||
|
||||
private NodeInformation parseNode(Path nodePath) {
|
||||
try (Stream<Path> indicesDirs = Files.list(nodePath.resolve("indices"))) {
|
||||
final StateFile state = getState(nodePath, "node");
|
||||
final List<IndexInformation> indices = indicesDirs
|
||||
.map(this::parseIndex)
|
||||
.sorted(Comparator.comparing(IndexInformation::indexName))
|
||||
.collect(Collectors.toList());
|
||||
return new NodeInformation(nodePath, indices, state);
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to list indices directories", e);
|
||||
}
|
||||
}
|
||||
|
||||
private StateFile getState(Path path, String stateFilePrefix) {
|
||||
final Path stateFile = findStateFile(path, stateFilePrefix);
|
||||
return stateFileParser.parse(stateFile);
|
||||
}
|
||||
|
||||
private IndexInformation parseIndex(Path path) {
|
||||
final String indexID = path.getFileName().toString();
|
||||
final StateFile state = getState(path, "state");
|
||||
try (Stream<Path> shardDirs = Files.list(path)) {
|
||||
final List<ShardInformation> shards = shardDirs
|
||||
.filter(Files::isDirectory)
|
||||
.filter(p -> p.getFileName().toString().matches("\\d+"))
|
||||
.filter(p -> Files.exists(p.resolve("index")))
|
||||
.map(this::getShardInformation)
|
||||
.sorted(Comparator.comparing(ShardInformation::name))
|
||||
.collect(Collectors.toList());
|
||||
return new IndexInformation(path, indexID, state, shards);
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to parse shard information", e);
|
||||
}
|
||||
}
|
||||
|
||||
private ShardInformation getShardInformation(Path path) {
|
||||
final ShardStats shardStats = shardReader.read(path);
|
||||
final StateFile state = getState(path, "state");
|
||||
return new ShardInformation(path, shardStats.documentsCount(), state, shardStats.minSegmentLuceneVersion());
|
||||
}
|
||||
|
||||
private Path findStateFile(Path stateDir, String stateFilePrefix) {
|
||||
try (Stream<Path> stateFiles = Files.list(stateDir.resolve(STATE_DIR_NAME))) {
|
||||
return stateFiles
|
||||
.filter(Files::isRegularFile)
|
||||
.filter(file -> file.getFileName().toString().startsWith(stateFilePrefix))
|
||||
.filter(file -> file.getFileName().toString().endsWith(STATE_FILE_EXTENSION))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new IndexerInformationParserException("No state file available in dir " + stateDir));
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to list state file of index" + stateDir, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFile;
|
||||
import org.graylog.shaded.opensearch2.org.opensearch.Version;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public record IndexInformation(@JsonIgnore Path path, String indexID, @JsonIgnore StateFile stateFile,
|
||||
List<ShardInformation> shards) {
|
||||
|
||||
@JsonProperty
|
||||
public String indexName() {
|
||||
return stateFile.document().keySet().stream().findFirst().orElseThrow(() -> new RuntimeException("Failed to read index name"));
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String indexVersionCreated() {
|
||||
final int versionValue = Integer.parseInt(indexSetting("index.version.created"));
|
||||
return Version.fromId(versionValue).toString();
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String creationDate() {
|
||||
|
||||
final long timestamp = Long.parseLong(indexSetting("index.creation_date"));
|
||||
return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDateTime().toString();
|
||||
}
|
||||
|
||||
private String indexSetting(String setting) {
|
||||
final Map<String, Object> index = (Map<String, Object>) stateFile.document().get(indexName());
|
||||
Map<String, Object> settings = (Map<String, Object>) index.get("settings");
|
||||
return (String) settings.get(setting);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{" +
|
||||
"indexID='" + indexID + '\'' +
|
||||
", indexName='" + indexName() + '\'' +
|
||||
", created='" + creationDate() + '\'' +
|
||||
", version='" + indexVersionCreated() + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public record IndexerDirectoryInformation(@JsonIgnore Path path, List<NodeInformation> nodes) {
|
||||
public static IndexerDirectoryInformation empty(Path path) {
|
||||
return new IndexerDirectoryInformation(path, Collections.emptyList());
|
||||
}
|
||||
|
||||
/**
|
||||
* The property name is matching the configuration property name, to minimize any confusion.
|
||||
* Jackson is by default serializing paths with file:/ prefix, so we are doing the conversion to string
|
||||
* here on our own to deliver only the real path value.
|
||||
*/
|
||||
@JsonProperty("opensearch_data_location")
|
||||
public String baseDir() {
|
||||
return path.toString();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFile;
|
||||
import org.graylog.shaded.opensearch2.org.opensearch.Version;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public record NodeInformation(@JsonIgnore java.nio.file.Path nodePath, List<IndexInformation> indices,
|
||||
@JsonIgnore StateFile stateFile) {
|
||||
@JsonProperty
|
||||
public String nodeVersion() {
|
||||
return Optional.ofNullable((Integer)stateFile.document().get("node_version"))
|
||||
.map(Version::fromId)
|
||||
.map(Version::toString)
|
||||
.orElseGet(this::parseFromIndices);
|
||||
}
|
||||
|
||||
private String parseFromIndices() {
|
||||
return indices.stream().map(IndexInformation::indexVersionCreated).distinct().findFirst().orElse(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.graylog.datanode.filesystem.index.statefile.StateFile;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.util.Version;
|
||||
|
||||
import java.util.Optional;
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public record ShardInformation(@JsonIgnore java.nio.file.Path path, int documentsCount, @JsonIgnore StateFile stateFile,
|
||||
@JsonIgnore Version minSegmentLuceneVersion) {
|
||||
|
||||
@JsonProperty
|
||||
public String name() {
|
||||
return "S" + path.getFileName().toString();
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String minLuceneVersion() {
|
||||
return Optional.ofNullable(minSegmentLuceneVersion).map(Version::toString).orElse(null);
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean primary() {
|
||||
return (boolean) stateFile.document().get("primary");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{" +
|
||||
"name=" + name() +
|
||||
", documentsCount=" + documentsCount +
|
||||
", minSegmentLuceneVersion=" + minSegmentLuceneVersion +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.indexreader;
|
||||
|
||||
public record ShardStats(java.nio.file.Path path, int documentsCount,
|
||||
org.graylog.shaded.opensearch2.org.apache.lucene.util.Version minSegmentLuceneVersion) {
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.indexreader;
|
||||
|
||||
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
public interface ShardStatsParser {
|
||||
ShardStats read(Path path) throws IncompatibleIndexVersionException;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.indexreader;
|
||||
|
||||
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
|
||||
import org.graylog.datanode.filesystem.index.IndexerInformationParserException;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.index.DirectoryReader;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.index.IndexFormatTooOldException;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.index.StandardDirectoryReader;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.store.Directory;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.store.FSDirectory;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.util.Version;
|
||||
|
||||
import javax.inject.Singleton;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
@Singleton
|
||||
public class ShardStatsParserImpl implements ShardStatsParser {
|
||||
@Override
|
||||
public ShardStats read(Path shardPath) throws IncompatibleIndexVersionException {
|
||||
try (Directory directory = FSDirectory.open(shardPath.resolve("index"))) {
|
||||
final StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(directory);
|
||||
final int documentsCount = getDocumentsCount(reader);
|
||||
final Version minSegmentLuceneVersion = reader.getSegmentInfos().getMinSegmentLuceneVersion();
|
||||
return new ShardStats(shardPath, documentsCount, minSegmentLuceneVersion);
|
||||
} catch (IndexFormatTooOldException e) {
|
||||
throw new IncompatibleIndexVersionException(e);
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to open index for read", e);
|
||||
}
|
||||
}
|
||||
|
||||
private int getDocumentsCount(StandardDirectoryReader reader) {
|
||||
// use IndexSearcher if you want to count documents smarter, filtering by field or query
|
||||
// IndexSearcher searcher = new IndexSearcher(reader);
|
||||
return reader.numDocs();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.statefile;
|
||||
|
||||
public record StateFile(java.nio.file.Path file, java.util.Map<String, Object> document) {
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.statefile;
|
||||
|
||||
import org.graylog.datanode.filesystem.index.IndexerInformationParserException;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
public interface StateFileParser {
|
||||
StateFile parse(Path file) throws IndexerInformationParserException;
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.filesystem.index.statefile;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.core.JsonParser;
|
||||
import com.fasterxml.jackson.core.StreamReadConstraints;
|
||||
import com.fasterxml.jackson.core.StreamReadFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
|
||||
import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
|
||||
import org.graylog.datanode.filesystem.index.IndexerInformationParserException;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.codecs.CodecUtil;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.store.IOContext;
|
||||
import org.graylog.shaded.opensearch2.org.apache.lucene.store.IndexInput;
|
||||
import org.graylog.shaded.opensearch2.org.opensearch.common.lucene.store.InputStreamIndexInput;
|
||||
import org.graylog2.jackson.TypeReferences;
|
||||
|
||||
import javax.inject.Singleton;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
@Singleton
|
||||
public class StateFileParserImpl implements StateFileParser {
|
||||
|
||||
private static final String STATE_FILE_CODEC = "state";
|
||||
private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 1;
|
||||
private static final int STATE_FILE_VERSION = 1;
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public StateFileParserImpl() {
|
||||
this.objectMapper = new ObjectMapper(createSmileFactory());
|
||||
}
|
||||
|
||||
private SmileFactory createSmileFactory() {
|
||||
final SmileFactory factory = new SmileFactory();
|
||||
// for now, this is an overhead, might make sense for web sockets
|
||||
factory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false);
|
||||
factory.configure(SmileFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...
|
||||
// Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.smile.SmileGenerator#close() method
|
||||
factory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
|
||||
factory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true);
|
||||
factory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(50000000).build());
|
||||
factory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true);
|
||||
return factory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StateFile parse(Path file) throws IndexerInformationParserException {
|
||||
try {
|
||||
return parseStateFile(file);
|
||||
} catch (IOException e) {
|
||||
throw new IndexerInformationParserException("Failed to parse state file", e);
|
||||
}
|
||||
}
|
||||
|
||||
private StateFile parseStateFile(Path file) throws IOException {
|
||||
final Path dir = file.getParent();
|
||||
final String filename = file.getFileName().toString();
|
||||
final org.graylog.shaded.opensearch2.org.apache.lucene.store.FSDirectory directory = org.graylog.shaded.opensearch2.org.apache.lucene.store.FSDirectory.open(dir);
|
||||
IndexInput indexInput = EndiannessReverserUtil.openInput(directory, filename, IOContext.DEFAULT);
|
||||
// We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
|
||||
CodecUtil.checksumEntireFile(indexInput);
|
||||
CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION);
|
||||
final int xcontentTypeValue = indexInput.readInt();
|
||||
long filePointer = indexInput.getFilePointer();
|
||||
long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
|
||||
try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
|
||||
final InputStreamIndexInput input = new InputStreamIndexInput(slice, contentSize);
|
||||
final Map<String, Object> readValue = objectMapper.readValue(input, TypeReferences.MAP_STRING_OBJECT);
|
||||
return new StateFile(file, readValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.rest;
|
||||
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
|
||||
|
||||
public record CompatibilityResult(String opensearchVersion,
|
||||
IndexerDirectoryInformation info,
|
||||
java.util.List<String> compatibilityErrors) {
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* Copyright (C) 2020 Graylog, Inc.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the Server Side Public License, version 1,
|
||||
* as published by MongoDB, Inc.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* Server Side Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the Server Side Public License
|
||||
* along with this program. If not, see
|
||||
* <http://www.mongodb.com/licensing/server-side-public-license>.
|
||||
*/
|
||||
package org.graylog.datanode.rest;
|
||||
|
||||
import org.graylog.datanode.configuration.DatanodeConfiguration;
|
||||
import org.graylog.datanode.filesystem.index.IncompatibleIndexVersionException;
|
||||
import org.graylog.datanode.filesystem.index.IndicesDirectoryParser;
|
||||
import org.graylog.datanode.filesystem.index.dto.IndexerDirectoryInformation;
|
||||
import org.graylog.datanode.filesystem.index.dto.NodeInformation;
|
||||
import org.graylog.datanode.rest.config.OnlyInSecuredNode;
|
||||
import org.graylog.shaded.opensearch2.org.opensearch.Version;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
@Path("/indices-directory")
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
public class IndicesDirectoryController {
|
||||
|
||||
private final DatanodeConfiguration datanodeConfiguration;
|
||||
private final IndicesDirectoryParser indicesDirectoryParser;
|
||||
|
||||
@Inject
|
||||
public IndicesDirectoryController(DatanodeConfiguration datanodeConfiguration, IndicesDirectoryParser indicesDirectoryParser) {
|
||||
this.datanodeConfiguration = datanodeConfiguration;
|
||||
this.indicesDirectoryParser = indicesDirectoryParser;
|
||||
}
|
||||
|
||||
@OnlyInSecuredNode
|
||||
@GET
|
||||
@Path("compatibility")
|
||||
public CompatibilityResult status() {
|
||||
final java.nio.file.Path dataTargetDir = datanodeConfiguration.datanodeDirectories().getDataTargetDir();
|
||||
final String opensearchVersion = datanodeConfiguration.opensearchDistributionProvider().get().version();
|
||||
try {
|
||||
final IndexerDirectoryInformation info = indicesDirectoryParser.parse(dataTargetDir);
|
||||
final Version currentVersion = Version.fromString(opensearchVersion);
|
||||
final List<String> compatibilityErrors = info.nodes().stream()
|
||||
.filter(node -> !isNodeCompatible(node, currentVersion))
|
||||
.map(node -> String.format(Locale.ROOT, "Current version %s of Opensearch is not compatible with index version %s", currentVersion, node.nodeVersion()))
|
||||
.toList();
|
||||
return new CompatibilityResult(opensearchVersion, info, compatibilityErrors);
|
||||
} catch (IncompatibleIndexVersionException e) {
|
||||
return new CompatibilityResult(opensearchVersion, new IndexerDirectoryInformation(dataTargetDir, Collections.emptyList()), Collections.singletonList(e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean isNodeCompatible(NodeInformation node, Version currentVersion) {
|
||||
final Version nodeVersion = Version.fromString(node.nodeVersion());
|
||||
return node.nodeVersion() == null || currentVersion.isCompatible(nodeVersion);
|
||||
}
|
||||
}
|
||||
@@ -24,5 +24,6 @@ public class RestBindings extends Graylog2Module {
|
||||
addSystemRestResource(StatusController.class);
|
||||
addSystemRestResource(LogsController.class);
|
||||
addSystemRestResource(ManagementController.class);
|
||||
addSystemRestResource(IndicesDirectoryController.class);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user