From ff630ad1c41a5b2b7c55a3eb2dbf7fc78fd28323 Mon Sep 17 00:00:00 2001 From: Jan Heise Date: Thu, 28 Mar 2024 13:22:04 +0100 Subject: [PATCH] DataNode: Setting the remote reindexing allowlist separately (#18805) * setting the whitelist separate from the host that we execute the command against. needs to be two settings because the whitelist can contain wildcards * adding the whitelist to the UI * fix import, fix text * fix test * changing whitelist -> allowlist * fixing merge conflicts --- .../management/OpensearchProcessService.java | 4 +-- .../RemoteReindexingMigrationAdapterOS2.java | 25 ++++++++++--------- .../migration/RemoteReindexParams.java | 9 +++++-- .../migration/RemoteReindexResource.java | 2 +- .../state/actions/MigrationActionsImpl.java | 3 ++- .../datanode/RemoteReindexAllowlistEvent.java | 4 ++- .../datanode/RemoteReindexRequest.java | 7 +++++- .../hooks/useRemoteReindexMigrationStatus.ts | 1 + .../RemoteReindexingMigration.test.tsx | 9 ++++--- .../remoteReindexing/MigrateExistingData.tsx | 21 +++++++++++++--- 10 files changed, 58 insertions(+), 27 deletions(-) diff --git a/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java b/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java index f36e252ad5..e58390b18b 100644 --- a/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java +++ b/data-node/src/main/java/org/graylog/datanode/management/OpensearchProcessService.java @@ -109,7 +109,7 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro switch (event.action()) { case ADD -> { this.process.stop(); - configure(Map.of("reindex.remote.whitelist", event.host())); // , "action.auto_create_index", "false")); + configure(Map.of("reindex.remote.allowlist", event.allowlist())); // , "action.auto_create_index", "false")); this.process.start(); } case REMOVE -> { @@ -165,7 +165,7 @@ public class OpensearchProcessService extends AbstractIdleService implements Pro this.configure(Map.of()); } - private void configure(Map additionalConfig) { + private void configure(Map additionalConfig) { final OpensearchConfiguration original = configurationProvider.get(); final var finalAdditionalConfig = new HashMap(); diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java index 6e8809cd3d..c2280d2235 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java @@ -69,6 +69,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.MalformedURLException; import java.net.URI; +import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; @@ -146,7 +147,7 @@ public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigr private void doStartMigration(RemoteReindexMigration migration, RemoteReindexRequest request) { try { - prepareCluster(request.uri()); + prepareCluster(request.allowlist()); createIndicesInNewCluster(migration); startAsyncTasks(migration, request); } catch (Exception e) { @@ -165,20 +166,20 @@ public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigr }); } - private void prepareCluster(URI uri) { + private void prepareCluster(String allowlistAsString) { final var activeNodes = getAllActiveNodeIDs(); - final String reindexSourceAddress = uri.getHost() + ":" + uri.getPort(); + List allowlist = Arrays.asList(allowlistAsString.split(",")); try { - verifyRemoteReindexAllowlistSetting(reindexSourceAddress); + verifyRemoteReindexAllowlistSetting(allowlist); } catch (RemoteReindexNotAllowedException e) { // this is expected state for fresh datanode cluster - there is no value configured in the reindex.remote.allowlist // we have to add it to the configuration and wait till the whole cluster restarts - allowReindexingFrom(reindexSourceAddress); + allowReindexingFrom(allowlist); waitForClusterRestart(activeNodes); } // verify again, just to be sure that all the configuration is in place and vali - verifyRemoteReindexAllowlistSetting(reindexSourceAddress); + verifyRemoteReindexAllowlistSetting(allowlist); } ReindexRequest createReindexRequest(final String index, final BytesReference query, URI uri, String username, String password) { @@ -205,8 +206,8 @@ public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigr } } - public void verifyRemoteReindexAllowlistSetting(String reindexSourceAddress) throws RemoteReindexNotAllowedException { - final String allowlistSetttingValue = client.execute((restHighLevelClient, requestOptions) -> { + public void verifyRemoteReindexAllowlistSetting(List allowlistEntries) throws RemoteReindexNotAllowedException { + final String allowlistSettingValue = client.execute((restHighLevelClient, requestOptions) -> { final ClusterGetSettingsRequest request = new ClusterGetSettingsRequest(); request.includeDefaults(true); final ClusterGetSettingsResponse settings = restHighLevelClient.cluster().getSettings(request, requestOptions); @@ -215,9 +216,9 @@ public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigr // the value is not proper json, just something like [localhost:9201]. It should be safe to simply use String.contains, // but there is maybe a chance for mismatches and then we'd have to parse the value - final boolean isRemoteReindexAllowed = !allowlistSetttingValue.contains(reindexSourceAddress); + final boolean isRemoteReindexAllowed = !allowlistEntries.stream().allMatch(entry -> allowlistSettingValue.contains(entry)); if (isRemoteReindexAllowed) { - final String message = "Failed to configure reindex.remote.allowlist setting in the datanode cluster. Current setting value: " + allowlistSetttingValue; + final String message = "Failed to configure reindex.remote.allowlist setting in the datanode cluster. Current setting value: " + allowlistSettingValue; LOG.error(message); throw new RemoteReindexNotAllowedException(message); } @@ -262,8 +263,8 @@ public class RemoteReindexingMigrationAdapterOS2 implements RemoteReindexingMigr } } - void allowReindexingFrom(final String host) { - eventBus.post(new RemoteReindexAllowlistEvent(host, RemoteReindexAllowlistEvent.ACTION.ADD)); + void allowReindexingFrom(final List allowlist) { + eventBus.post(new RemoteReindexAllowlistEvent(allowlist, RemoteReindexAllowlistEvent.ACTION.ADD)); } List getAllIndicesFrom(final URI uri, final String username, final String password) throws MalformedURLException { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexParams.java b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexParams.java index ddbe90bddf..ad9371485f 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexParams.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexParams.java @@ -19,6 +19,11 @@ package org.graylog.plugins.views.storage.migration; import java.net.URI; import java.util.List; -public record RemoteReindexParams(URI hostname, String user, String password, List indices, - boolean synchronous, int threadsCount) { +public record RemoteReindexParams(String allowlist, + URI hostname, + String user, + String password, + List indices, + boolean synchronous, + int threadsCount) { } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexResource.java b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexResource.java index 9139643ea1..98476d8247 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/RemoteReindexResource.java @@ -56,7 +56,7 @@ public class RemoteReindexResource { @RequiresPermissions(RestPermissions.DATANODE_MIGRATION) @ApiOperation(value = "by remote reindexing", notes = "configure the host/credentials you want to use to migrate data from") public RemoteReindexMigration migrate(@ApiParam(name = "remote configuration") @NotNull @Valid RemoteReindexParams params) { - final RemoteReindexRequest req = new RemoteReindexRequest(params.hostname(), params.user(), params.password(), params.indices(), params.threadsCount()); + final RemoteReindexRequest req = new RemoteReindexRequest(params.allowlist(), params.hostname(), params.user(), params.password(), params.indices(), params.threadsCount()); return migrationService.start(req); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java index 6899b08107..c321e1d176 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java @@ -196,12 +196,13 @@ public class MigrationActionsImpl implements MigrationActions { @Override public void startRemoteReindex() { + final String allowlist = getStateMachineContext().getActionArgumentOpt("allowlist", String.class).orElseThrow(() -> new NullPointerException("allowlist has tp be provided")); final URI hostname = Objects.requireNonNull(URI.create(getStateMachineContext().getActionArgument("hostname", String.class)), "hostname has to be provided"); final String user = getStateMachineContext().getActionArgumentOpt("user", String.class).orElse(null); final String password = getStateMachineContext().getActionArgumentOpt("password", String.class).orElse(null); final List indices = getStateMachineContext().getActionArgumentOpt("indices", List.class).orElse(Collections.emptyList()); // todo: generics! final int threadsCount = getStateMachineContext().getActionArgumentOpt("threads", Integer.class).orElse(4); - final RemoteReindexMigration migration = migrationService.start(new RemoteReindexRequest(hostname, user, password, indices, threadsCount)); + final RemoteReindexMigration migration = migrationService.start(new RemoteReindexRequest(allowlist, hostname, user, password, indices, threadsCount)); final String migrationID = migration.id(); getStateMachineContext().addExtendedState(MigrationStateMachineContext.KEY_MIGRATION_ID, migrationID); } diff --git a/graylog2-server/src/main/java/org/graylog2/datanode/RemoteReindexAllowlistEvent.java b/graylog2-server/src/main/java/org/graylog2/datanode/RemoteReindexAllowlistEvent.java index 4962dd8a20..1f48300bd1 100644 --- a/graylog2-server/src/main/java/org/graylog2/datanode/RemoteReindexAllowlistEvent.java +++ b/graylog2-server/src/main/java/org/graylog2/datanode/RemoteReindexAllowlistEvent.java @@ -16,7 +16,9 @@ */ package org.graylog2.datanode; -public record RemoteReindexAllowlistEvent(String host, ACTION action) { +import java.util.List; + +public record RemoteReindexAllowlistEvent(List allowlist, ACTION action) { public enum ACTION { ADD, REMOVE } diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/datanode/RemoteReindexRequest.java b/graylog2-server/src/main/java/org/graylog2/indexer/datanode/RemoteReindexRequest.java index 96f826217f..1cb44c0dc9 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/datanode/RemoteReindexRequest.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/datanode/RemoteReindexRequest.java @@ -19,5 +19,10 @@ package org.graylog2.indexer.datanode; import java.net.URI; import java.util.List; -public record RemoteReindexRequest(URI uri, String username, String password, List indices, int threadsCount) { +public record RemoteReindexRequest(String allowlist, + URI uri, + String username, + String password, + List indices, + int threadsCount) { } diff --git a/graylog2-web-interface/src/components/datanode/hooks/useRemoteReindexMigrationStatus.ts b/graylog2-web-interface/src/components/datanode/hooks/useRemoteReindexMigrationStatus.ts index 9e40fb0425..fd1ac3b2ed 100644 --- a/graylog2-web-interface/src/components/datanode/hooks/useRemoteReindexMigrationStatus.ts +++ b/graylog2-web-interface/src/components/datanode/hooks/useRemoteReindexMigrationStatus.ts @@ -39,6 +39,7 @@ export type RemoteReindexMigration = { } export type RemoteReindexRequest = { + allowlist: string, hostname: string, password: string, indices: string[], diff --git a/graylog2-web-interface/src/components/datanode/migrations/RemoteReindexingMigration.test.tsx b/graylog2-web-interface/src/components/datanode/migrations/RemoteReindexingMigration.test.tsx index 6a408e80b5..ed1133220b 100644 --- a/graylog2-web-interface/src/components/datanode/migrations/RemoteReindexingMigration.test.tsx +++ b/graylog2-web-interface/src/components/datanode/migrations/RemoteReindexingMigration.test.tsx @@ -70,7 +70,8 @@ jest.mock('components/datanode/hooks/useDataNodes', () => jest.fn(() => ({ count: 0, }, }, - refetch: () => {}, + refetch: () => { + }, isInitialLoading: false, error: null, }))); @@ -83,7 +84,8 @@ const renderStep = (_state: MigrationStateItem) => { response: null, } as MigrationState); - render( ({} as MigrationState)} currentStep={getCurrentStep(_state)} />); + render( ({} as MigrationState)} + currentStep={getCurrentStep(_state)} />); }; describe('RemoteReindexingMigration', () => { @@ -131,7 +133,8 @@ describe('RemoteReindexingMigration', () => { name: /5. Migrate existing data/i, }); - await screen.findByLabelText(/Cluster URI/); + await screen.findByLabelText(/URI of the host/); + await screen.findByLabelText(/Allowlist/); await screen.findByLabelText(/Username/); await screen.findByLabelText(/Password/); }); diff --git a/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx b/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx index 0f9de85f10..1e3236290a 100644 --- a/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx +++ b/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx @@ -42,7 +42,7 @@ export type RemoteReindexCheckConnection = { const MigrateExistingData = ({ currentStep, onTriggerStep }: MigrationStepComponentProps) => { const [nextSteps, setNextSteps] = useState(['CHECK_REMOTE_INDEXER_CONNECTION']); - const [errorMessage, setErrrorMessage] = useState(null); + const [errorMessage, setErrrorMessage] = useState(null); const [isLoading, setIsLoading] = useState(false); const [availableIndices, setAvailableIndices] = useState([]); const [selectedIndices, setSelectedIndices] = useState([]); @@ -100,6 +100,7 @@ const MigrateExistingData = ({ currentStep, onTriggerStep }: MigrationStepCompon }; const initialValues: RemoteReindexRequest = { + allowlist: '', hostname: '', user: '', password: '', @@ -108,15 +109,25 @@ const MigrateExistingData = ({ currentStep, onTriggerStep }: MigrationStepCompon }; return ( - {}}> + { + }}> {({ values, setFieldValue, }) => (
+ handleChange(e, setFieldValue)} + required /> ) : ( - + )} )}