diff --git a/changelog/unreleased/pr-17707.toml b/changelog/unreleased/pr-17707.toml new file mode 100644 index 0000000000..084d34aa58 --- /dev/null +++ b/changelog/unreleased/pr-17707.toml @@ -0,0 +1,4 @@ +type = "a" +message = "Fallback guide on how to do manual migration into the DataNode" + +pulls = ["17707"] diff --git a/data-node/migration/Manual-Migration.md b/data-node/migration/Manual-Migration.md new file mode 100644 index 0000000000..e5c87052f5 --- /dev/null +++ b/data-node/migration/Manual-Migration.md @@ -0,0 +1,174 @@ +# Manual Migration Guide + +**Caveat**: preliminary - and only to have a minimal basis to find out which modifications are necessary at the current codebase +and to research/test possible improvements for a better user experience or find out the limits what we can actually do. +Some of these steps could be useful in the future for PSO or support to migrate manually or fix problems that occurred during migration. + +**Also: if you try to use this guide for a manual migration of a real prod system, all the preliniaries regarding certificates etc. probably +don't match your use case. Please adjust accordingly!** + +#### Testing during development + +Create/include a locally generated Docker image for Graylog or the DataNode by using the `create-docker-images.sh` + +At the time of writing this, if you test/develop on macOS and you're struggling with setting `vm.max_map_count` for Docker, +you're running the latest Docker version, which is incompatible with macOS Sonoma. Your only chance is to downgrade Docker. + +## Things to watch out for +The following is a list of items that I did not put in much effort/time to test but should be kept in mind to avoid potential issues: +- keep hostnames as they are +- keep the cluster name + +Otherwise you might create issues with existing meta data in OpenSearch. + + +In a lot of cases, missing a step or having an error, you have no other chance as to start over completely. Which might not be +possible in PROD environments. So make sure you have backups ;-) + +## Migrating an existing OpenSearch 2.x or 1.3.x cluster + +This is a preliminary guide to migrate an existing OpenSearch 2.x or 1.3.x cluster into a DataNode cluster. +Basis was the `cluster` from our `docker compose` github repository. But we only use one MongoDB and one Graylog node. + +It is based on Docker to be able to reproduce all the steps consistently. Real installations will surely +differ. The steps can be also used to migrate a OS packages install. + +## Migrating Elasticsearch 7.10 + +see additional info at the end of this document + +### Contents of this directory: + +- `docker-compose.yml` - the Docker compose file we're using and modifying in between steps +- `env` - the Environment variables read by `docker-compose.yml` for hashes and the admin password +- `cert.sh` - a script to create certificates for the OpenSearch cluster +- `custom-opensearch.yml` - certificate related OpenSearch configuration +- `opensearch-security-config.yml` - the OpenSearch security config that we`re going to replace during a migration +- `datanode-security-config.yml` - the default DataNode security config that we'll use as a replacement + +### Migration Steps + +You should be able to always start over by doing a `docker compose down -v` and taking back any modifications on the files. + +#### The file `cert.sh` +Use the `cert.sh` script to generate the certificate certificates for the OpenSearch cluster. Use "password" as the password +for certificate related stuff so that it matches the other scripts/files. + +For a cluster that uses no certificates, don't create certificates. Remove the following lines +from each OpenSearch service in `docker-compose.yml`: + +``` +- "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" +- "./node?.pem:/usr/share/opensearch/config/node.pem" +- "./node?-key.pem:/usr/share/opensearch/config/node-key.pem" +- "./admin.pem:/usr/share/opensearch/config/admin.pem" +- "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" +- "./keystore.jks:/usr/share/opensearch/config/keystore.jks" +- "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" +``` +Replace the `?` with 1-3 depending on the service. + +And remove the cert config from the graylog service by deleting the following lines: + +``` + GRAYLOG_CA_KEYSTORE_FILE: "/usr/share/graylog/data/keystore.jks" + GRAYLOG_CA_PASSWORD: "password" +``` +and +``` + - "./keystore.jks:/usr/share/graylog/data/keystore.jks" +``` + +#### The file `env` +Modify the `env` file like with the regular docker compose examples that we provide. Rename it to `.env` . + + + +#### The file `datanode-security-config.yml` +Convert the `GRAYLOG_PASSWORD_SECRET` to base64 e.g. by doing `echo "The password secret you chose" | base64` and +put it into line 131 of `datanode-security-config.yml` + +### OpenSearch 1.3.x + +To run this with OpenSearch 1.3.x, replace the used Docker image `2.10.0` with `1.3.1` for all 3 services. + +### Create your containers + +`docker compose create` + +### Start Graylog with OpenSearch + +Start your cluster `docker compose up -d mongodb opensearch1 opensearch2 opensearch3 graylog1`. Create an input, +ingest some data. etc. + +Stop OpenSearch and Graylog with `docker compose stop graylog1 opensearch1 opensearch2 opensearch3`. + +Modify all three OpenSearch services in `docker-compose.yml` to look like this: +``` + - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" +# - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" +``` + +So basically, switch out the default security config that uses a user/role based model to one that uses JWT authentication +which is the mechanism we use with the DataNode. + +Bring the OpenSearch cluster back on: + +`docker compose create opensearch1 opensearch2 opensearch3` + +`docker compose up -d opensearch1 opensearch2 opensearch3` + +You can check the logs or test via `curl` that OpenSearch is back. + +Use `docker ps` to get the ID of one of the nodes. `docker exec -it bash` into the container. +Run `cd /usr/share/opensearch/plugins/opensearch-security/tools` and `./securityadmin.sh -f /usr/share/opensearch/config/opensearch-security/config.yml -icl -h opensearch1 -nhnv -cacert ../../../config/root-ca.pem -cert ../../../config/admin.pem -key ../../../config/admin-key.pem` +to make OpenSearch reload the security data. + +This step is necessary so that the OpenSearch data directory with the security indices etc. contain the JWT auth data before we attach the data ddirectory to the DataNode. + +Stop OpenSearch using `docker compose stop opensearch1 opensearch2 opensearch3` + +Modify your graylog service so it will connect to DataNodes and show the Preflight config, it should look like this: + +``` +# GRAYLOG_CA_KEYSTORE_FILE: "/usr/share/graylog/data/keystore.jks" +# GRAYLOG_CA_PASSWORD: "password" +# GRAYLOG_ELASTICSEARCH_HOSTS: "https://admin:admin@opensearch1:9200,https://admin:admin@opensearch2:9200,https://admin:admin@opensearch3:9200" + GRAYLOG_ENABLE_PREFLIGHT_WEB: "true" +``` + +Issue `docker compose create graylog1` and `docker compose up -d graylog1 datanode1 datanode2 datanode3`. +Log into the Preflight UI at `http://localhost:9000` with the auth credentials from the end of the logs at `docker compose logs -f graylog1` + +Provision certificates for the DataNodes, they should start up and get green. Resume the Graylog startup. +For now, nothing should happen - this will be addressed shortly in dev. Stop graylog with `docker compose stop graylog1`. + +Disable the Preflight UI in `docker-compose.yml` by adding a `#`: +``` +# GRAYLOG_ENABLE_PREFLIGHT_WEB: "true" +``` +Use `docker compose create graylog1` and `docker compose up -d graylog1` to bring graylog up again. + +Now you should be able the log into graylog at `http://localhost:9000` with your regular credentials and have a running +configuration with DataNodes. + + +## Migrating Elasticsearch 7.10 + +Migration from Elasticsearch 7.10 needs an additional step. ES 7.10 does not understand JWT authentication. +So you have to first migrate to OpenSearch before running the update of the security information. +Look at the supplied `es710-docker-compose.yml` as an example. Please note that except for the servicename, I changed the cluster name +and hostnames etc. to `opensearch`. In a regular setting, it would be the other way around and you would have to pull the +elasticsearch names through the whole process into the DataNode. + +Start the Elasticsearch cluster, add some data etc. `docker compose up -d elasticsearch1 elasticsearch2 elasticsearch3` +stop it again `docker compose stop elasticsearch1 elasticsearch2 elasticsearch3`. + +Start the OpenSearch cluster in place of the Elasticsearch cluster. It points to the same data directory. +`docker compose up -d opensearch1 opensearch2 opensearch3`. The `es710-docker-compose.yml` already points to the +security config with the JWT auth settings. Make sure you added the correct bas64 encoded secret. + +Run the `securityadmin.sh` as described above and just follow the steps for an OpenSearch 1.3 migration as described. + +Please also note that the ElasticSearch example does not contain any certificates for ElasticSearch but uses generated certificates +once you started the OpenSearch cluster. diff --git a/data-node/migration/cert.sh b/data-node/migration/cert.sh new file mode 100644 index 0000000000..295f357dee --- /dev/null +++ b/data-node/migration/cert.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Root CA +openssl genrsa -out root-ca-key.pem 2048 +openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=root" -out root-ca.pem -days 730 +# Admin cert +openssl genrsa -out admin-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in admin-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out admin-key.pem +openssl req -new -key admin-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=A" -out admin.csr +openssl x509 -req -in admin.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out admin.pem -days 730 +# Node cert 1 +openssl genrsa -out node1-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in node1-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node1-key.pem +openssl req -new -key node1-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=opensearch1" -out node1.csr +echo 'subjectAltName=DNS:opensearch1' > node1.ext +openssl x509 -req -in node1.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node1.pem -days 730 -extfile node1.ext +# Node cert 2 +openssl genrsa -out node2-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in node2-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node2-key.pem +openssl req -new -key node2-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=opensearch2" -out node2.csr +echo 'subjectAltName=DNS:opensearch2' > node2.ext +openssl x509 -req -in node2.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node2.pem -days 730 -extfile node2.ext +# Node cert 3 +openssl genrsa -out node3-key-temp.pem 2048 +openssl pkcs8 -inform PEM -outform PEM -in node3-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node3-key.pem +openssl req -new -key node3-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=opensearch3" -out node3.csr +echo 'subjectAltName=DNS:opensearch3' > node3.ext +openssl x509 -req -in node3.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node3.pem -days 730 -extfile node3.ext +# Client cert +#openssl genrsa -out client-key-temp.pem 2048 +#openssl pkcs8 -inform PEM -outform PEM -in client-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out client-key.pem +#openssl req -new -key client-key.pem -subj "/C=CA/ST=ONTARIO/L=TORONTO/O=ORG/OU=UNIT/CN=client.dns.a-record" -out client.csr +#echo 'subjectAltName=DNS:client.dns.a-record' > client.ext +#openssl x509 -req -in client.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out client.pem -days 730 -extfile client.ext +# Cleanup +rm admin-key-temp.pem +rm admin.csr +rm node1-key-temp.pem +rm node1.csr +rm node1.ext +rm node2-key-temp.pem +rm node2.csr +rm node2.ext +rm node3-key-temp.pem +rm node3.csr +rm node3.ext +#rm client-key-temp.pem +#rm client.csr +#rm client.ext + + +keytool -import -trustcacerts -alias opensearch1 -file node1.pem -keystore keystore.jks +keytool -import -trustcacerts -alias opensearch2 -file node2.pem -keystore keystore.jks +keytool -import -trustcacerts -alias opensearch3 -file node3.pem -keystore keystore.jks +keytool -import -trustcacerts -alias admin -file admin.pem -keystore keystore.jks +keytool -import -trustcacerts -alias root -file root-ca.pem -keystore keystore.jks + diff --git a/data-node/migration/create-docker-images.sh b/data-node/migration/create-docker-images.sh new file mode 100755 index 0000000000..34d97c2fc3 --- /dev/null +++ b/data-node/migration/create-docker-images.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# script to build local docker images for graylog and the datanode +# +# put this script in the root dir where you have graylog checked out, also check out the graylog-docker repo in the same dir. +# you should have the following structure: +# +# ls [enter] +# create-docker-images.sh +# graylog-docker +# graylog-project-internal +# graylog-project-repos +# +# run the script after a "mvn -DskipTests clean package" to build the images +# +# put the following lines in your .env file to reference locally created docker images +# DATANODE_IMAGE=graylog/graylog-datanode:local +# GRAYLOG_IMAGE=graylog/graylog:local +# + +cd graylog-docker + +cp `ls -1 ../graylog-project-internal/target/artifacts/graylog-datanode/graylog-datanode-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` . +docker build -t graylog/graylog-datanode:local --build-arg JAVA_VERSION_MAJOR=17 --build-arg LOCAL_BUILD_TGZ=`ls -1 graylog-datanode-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` -f docker/datanode/Dockerfile . +rm `ls -1 graylog-datanode-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` + +cp `ls -1 ../graylog-project-internal/target/artifacts/graylog-enterprise/graylog-enterprise-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` . +docker build -t graylog/graylog:local --build-arg JAVA_VERSION_MAJOR=17 --build-arg LOCAL_BUILD_TGZ=`ls -1 graylog-enterprise-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` -f docker/enterprise/Dockerfile . +rm `ls -1 graylog-enterprise-6.0.0-SNAPSHOT-*-linux-x64.tar.gz|tail -1` + +cd .. diff --git a/data-node/migration/custom-opensearch.yml b/data-node/migration/custom-opensearch.yml new file mode 100644 index 0000000000..23f9c9c87c --- /dev/null +++ b/data-node/migration/custom-opensearch.yml @@ -0,0 +1,30 @@ +plugins.security.allow_unsafe_democertificates: true +plugins.security.ssl.transport.pemcert_filepath: node.pem +plugins.security.ssl.transport.pemkey_filepath: node-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.transport.resolve_hostname: false +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemcert_filepath: node.pem +plugins.security.ssl.http.pemkey_filepath: node-key.pem +plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem +plugins.security.authcz.admin_dn: + - 'CN=A,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' +plugins.security.nodes_dn: + - 'CN=opensearch1,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' + - 'CN=opensearch2,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' + - 'CN=opensearch3,OU=UNIT,O=ORG,L=TORONTO,ST=ONTARIO,C=CA' +#plugins.security.audit.type: internal_opensearch +plugins.security.allow_default_init_securityindex: true +plugins.security.system_indices.enabled: true + +plugins.security.ssl.transport.truststore_type: JKS +plugins.security.ssl.transport.truststore_filepath: keystore.jks +plugins.security.ssl.transport.truststore_password: password + +plugins.security.ssl.http.truststore_type: JKS +plugins.security.ssl.http.truststore_filepath: keystore.jks +plugins.security.ssl.http.truststore_password: password + +logger.org.opensearch: debug + diff --git a/data-node/migration/datanode-security-config.yml b/data-node/migration/datanode-security-config.yml new file mode 100644 index 0000000000..daf87ed876 --- /dev/null +++ b/data-node/migration/datanode-security-config.yml @@ -0,0 +1,247 @@ +--- + +# This is the main OpenSearch Security configuration file where authentication +# and authorization is defined. +# +# You need to configure at least one authentication domain in the authc of this file. +# An authentication domain is responsible for extracting the user credentials from +# the request and for validating them against an authentication backend like Active Directory for example. +# +# If more than one authentication domain is configured the first one which succeeds wins. +# If all authentication domains fail then the request is unauthenticated. +# In this case an exception is thrown and/or the HTTP status is set to 401. +# +# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect +# the roles from a given backend for the authenticated user. +# +# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both. +# http_enabled: true +# transport_enabled: true +# +# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to +# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated. +# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous" +# and one role named "anonymous_backendrole". +# If you enable anonymous authentication all HTTP authenticators will not challenge. +# +# +# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert" +# first and the challenging one last. +# Because it's not possible to challenge a client with two different authentication methods (for example +# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation +# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request. +# +# Default value of the challenge flag is true. +# +# +# HTTP +# basic (challenging) +# proxy (not challenging, needs xff) +# kerberos (challenging) +# clientcert (not challenging, needs https) +# jwt (not challenging) +# host (not challenging) #DEPRECATED, will be removed in a future version. +# host based authentication is configurable in roles_mapping + +# Authc +# internal +# noop +# ldap + +# Authz +# ldap +# noop + + + +_meta: + type: "config" + config_version: 2 + +config: + dynamic: + # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index + # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default) + # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently + #filtered_alias_mode: warn + #do_not_fail_on_forbidden: false + #kibana: + # Kibana multitenancy + #multitenancy_enabled: true + #server_username: kibanaserver + #index: '.kibana' + http: + anonymous_auth_enabled: false + xff: + enabled: false + internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern + #internalProxies: '.*' # trust all internal proxies, regex pattern + #remoteIpHeader: 'x-forwarded-for' + ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help + ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For + ###### and here https://tools.ietf.org/html/rfc7239 + ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve + authc: + kerberos_auth_domain: + http_enabled: false + transport_enabled: false + order: 6 + http_authenticator: + type: kerberos + challenge: true + config: + # If true a lot of kerberos/security related debugging output will be logged to standard out + krb_debug: false + # If true then the realm will be stripped from the user name + strip_realm_from_principal: true + authentication_backend: + type: noop + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: false + transport_enabled: false + order: 4 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + proxy_auth_domain: + description: "Authenticate via proxy" + http_enabled: false + transport_enabled: false + order: 3 + http_authenticator: + type: proxy + challenge: false + config: + user_header: "x-proxy-user" + roles_header: "x-proxy-roles" + authentication_backend: + type: noop + jwt_auth_domain: + description: "Authenticate via Json Web Token" + http_enabled: true + transport_enabled: true + order: 0 + http_authenticator: + type: jwt + challenge: false + config: + signing_key: "base64 encoded GRAYLOG_PASSWORD_SECRET from .env file" + jwt_header: "Authorization" + jwt_url_parameter: null + roles_key: "os_roles" + subject_key: null + authentication_backend: + type: noop + clientcert_auth_domain: + description: "Authenticate via SSL client certificates" + http_enabled: false + transport_enabled: false + order: 2 + http_authenticator: + type: clientcert + config: + username_attribute: cn #optional, if omitted DN becomes username + challenge: false + authentication_backend: + type: noop + ldap: + description: "Authenticate via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + order: 5 + http_authenticator: + type: basic + challenge: false + authentication_backend: + # LDAP authentication backend (authenticate users against a LDAP or Active Directory) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(sAMAccountName={0})' + # Use this attribute from the user as username (if not set then DN is used) + username_attribute: null + authz: + roles_from_myldap: + description: "Authorize via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + rolebase: 'ou=groups,dc=example,dc=com' + # Filter to search for roles (currently in the whole subtree beneath rolebase) + # {0} is substituted with the DN of the user + # {1} is substituted with the username + # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute + rolesearch: '(member={0})' + # Specify the name of the attribute which value should be substituted with {2} above + userroleattribute: null + # Roles as an attribute of the user entry + userrolename: disabled + #userrolename: memberOf + # The attribute in a role entry containing the name of that role, Default is "name". + # Can also be "dn" to use the full DN as rolename. + rolename: cn + # Resolve nested roles transitive (roles which are members of other roles and so on ...) + resolve_nested_roles: true + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(uid={0})' + # Skip users matching a user name, a wildcard or a regex pattern + #skip_users: + # - 'cn=Michael Jackson,ou*people,o=TEST' + # - '/\S*/' + roles_from_another_ldap: + description: "Authorize via another Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + type: ldap + #config goes here ... + # auth_failure_listeners: + # ip_rate_limiting: + # type: ip + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 + # internal_authentication_backend_limiting: + # type: username + # authentication_backend: intern + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 diff --git a/data-node/migration/docker-compose.yml b/data-node/migration/docker-compose.yml new file mode 100644 index 0000000000..8992cddbe4 --- /dev/null +++ b/data-node/migration/docker-compose.yml @@ -0,0 +1,234 @@ +# based on files in: +# https://github.com/Graylog2/docker-compose +# https://opensearch.org/docs/latest/security/configuration/generate-certificates/ +# see additional cert.sh for certificate creation and custom-opensearch.yml for opensearch configuration + +version: "3.8" + +services: + mongodb: + hostname: "mongodb" + image: "mongo:5.0" + ports: + - "27017:27017" + volumes: + - "mongodb-data:/data/db" + + opensearch1: + image: "opensearchproject/opensearch:2.10.0" + hostname: "opensearch1" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch1" + - "node.name=opensearch1" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch2,opensearch3" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ports: + - "9200:9200" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-01:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node1.pem:/usr/share/opensearch/config/node.pem" + - "./node1-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" + - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" +# - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + + opensearch2: + image: "opensearchproject/opensearch:2.10.0" + hostname: "opensearch2" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch2" + - "node.name=opensearch2" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch1,opensearch3" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-02:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node2.pem:/usr/share/opensearch/config/node.pem" + - "./node2-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" + - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" +# - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + + opensearch3: + image: "opensearchproject/opensearch:2.10.0" + hostname: "opensearch3" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch3" + - "node.name=opensearch3" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch1,opensearch2" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-03:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node3.pem:/usr/share/opensearch/config/node.pem" + - "./node3-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" + - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" +# - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + + datanode1: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch1 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8999:8999/tcp" # DataNode API + - "9200:9200" + volumes: + - "graylog-datanode-01:/var/lib/graylog-datanode" + - "opensearch-data-01:/var/lib/graylog-datanode/opensearch/data" + + datanode2: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch2 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8998:8999/tcp" # DataNode API + volumes: + - "graylog-datanode-02:/var/lib/graylog-datanode" + - "opensearch-data-02:/var/lib/graylog-datanode/opensearch/data" + + datanode3: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch3 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8997:8999/tcp" # DataNode API + volumes: + - "graylog-datanode-03:/var/lib/graylog-datanode" + - "opensearch-data-03:/var/lib/graylog-datanode/opensearch/data" + + graylog1: + image: "${GRAYLOG_IMAGE:-graylog/graylog:5.2.0}" + hostname: "graylog1" + depends_on: + - "mongodb" + entrypoint: "/docker-entrypoint.sh" + environment: + GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id" + GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_HTTP_EXTERNAL_URI: "http://127.0.0.1:9000/" + GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog" + GRAYLOG_REPORT_DISABLE_SANDBOX: "true" + GRAYLOG_CA_KEYSTORE_FILE: "/usr/share/graylog/data/keystore.jks" + GRAYLOG_CA_PASSWORD: "password" + GRAYLOG_ELASTICSEARCH_HOSTS: "https://admin:admin@opensearch1:9200,https://admin:admin@opensearch2:9200,https://admin:admin@opensearch3:9200" +# GRAYLOG_ENABLE_PREFLIGHT_WEB: "true" + ports: + - "9000:9000" # Graylog web interface and REST API + - "1514:1514" # Syslog TCP + - "1514:1514/udp" # Syslog UDP + - "12201:12201" # GELF TCP + - "12201:12201/udp" # GELF UDP + volumes: + - "graylog-data-01:/usr/share/graylog/data/data" + - "graylog-journal-01:/usr/share/graylog/data/journal" + - "./keystore.jks:/usr/share/graylog/data/keystore.jks" + +volumes: + graylog-datanode-01: + graylog-datanode-02: + graylog-datanode-03: + graylog-data-01: + graylog-journal-01: + mongodb-data: + opensearch-data-01: + opensearch-data-02: + opensearch-data-03: diff --git a/data-node/migration/env b/data-node/migration/env new file mode 100644 index 0000000000..1be738ba92 --- /dev/null +++ b/data-node/migration/env @@ -0,0 +1,14 @@ +# You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. +# Generate one by using for example: pwgen -N 1 -s 96 +# ATTENTION: This value must be the same on all Graylog nodes in the cluster. +# Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens) +GRAYLOG_PASSWORD_SECRET= + +# You MUST specify a hash password for the root user (which you only need to initially set up the +# system and in case you lose connectivity to your authentication backend) +# This password cannot be changed using the API or via the web interface. If you need to change it, +# modify it in this file. +# Create one by using for example: echo -n yourpassword | shasum -a 256 +# and put the resulting hash value into the following line +# CHANGE THIS! +GRAYLOG_ROOT_PASSWORD_SHA2= diff --git a/data-node/migration/es710-docker-compose.yml b/data-node/migration/es710-docker-compose.yml new file mode 100644 index 0000000000..05c8964424 --- /dev/null +++ b/data-node/migration/es710-docker-compose.yml @@ -0,0 +1,287 @@ +# based on files in: +# https://github.com/Graylog2/docker-compose +# https://opensearch.org/docs/latest/security/configuration/generate-certificates/ +# see additional cert.sh for certificate creation and custom-opensearch.yml for opensearch configuration + +version: "3.8" + +services: + mongodb: + hostname: "mongodb" + image: "mongo:5.0" + ports: + - "27017:27017" + volumes: + - "mongodb-data:/data/db" + + elasticsearch1: + image: "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" + hostname: "opensearch1" + environment: + node.name: "opensearch1" + cluster.name: "datanode-cluster" + discovery.seed_hosts: "opensearch2,opensearch3" + cluster.initial_master_nodes: "opensearch1,opensearch2,opensearch3" + bootstrap.memory_lock: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" + ports: + - "9200:9200" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - "opensearch-data-01:/usr/share/elasticsearch/data" + + elasticsearch2: + image: "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" + hostname: "opensearch2" + environment: + node.name: "opensearch2" + cluster.name: "datanode-cluster" + discovery.seed_hosts: "opensearch1,opensearch3" + cluster.initial_master_nodes: "opensearch1,opensearch2,opensearch3" + bootstrap.memory_lock: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - "opensearch-data-02:/usr/share/elasticsearch/data" + + elasticsearch3: + image: "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" + hostname: "opensearch3" + environment: + node.name: "opensearch3" + cluster.name: "datanode-cluster" + discovery.seed_hosts: "opensearch1,opensearch2" + cluster.initial_master_nodes: "opensearch1,opensearch2,opensearch3" + bootstrap.memory_lock: "true" + ES_JAVA_OPTS: "-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - "opensearch-data-03:/usr/share/elasticsearch/data" + + opensearch1: + image: "opensearchproject/opensearch:1.3.1" + hostname: "opensearch1" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch1" + - "node.name=opensearch1" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch2,opensearch3" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ports: + - "9200:9200" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-01:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node1.pem:/usr/share/opensearch/config/node.pem" + - "./node1-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" +# - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + + opensearch2: + image: "opensearchproject/opensearch:1.3.1" + hostname: "opensearch2" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch2" + - "node.name=opensearch2" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch1,opensearch3" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-02:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node2.pem:/usr/share/opensearch/config/node.pem" + - "./node2-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" +# - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + + opensearch3: + image: "opensearchproject/opensearch:1.3.1" + hostname: "opensearch3" + environment: + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "network.host=opensearch3" + - "node.name=opensearch3" + - "cluster.name=datanode-cluster" + - "network.bind_host=_site_" + - "discovery.seed_hosts=opensearch1,opensearch2" + - "cluster.initial_master_nodes=opensearch1,opensearch2,opensearch3" + - "bootstrap.memory_lock=true" + - "action.auto_create_index=false" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "plugins.security.allow_unsafe_democertificates=true" + # - "plugins.security.ssl.transport.enforce_hostname_verification=false" + # - "plugins.security.ssl.transport.resolve_hostname=false" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - "opensearch-data-03:/usr/share/opensearch/data" + - "./root-ca.pem:/usr/share/opensearch/config/root-ca.pem" + - "./node3.pem:/usr/share/opensearch/config/node.pem" + - "./node3-key.pem:/usr/share/opensearch/config/node-key.pem" + - "./admin.pem:/usr/share/opensearch/config/admin.pem" + - "./admin-key.pem:/usr/share/opensearch/config/admin-key.pem" + - "./custom-opensearch.yml:/usr/share/opensearch/config/opensearch.yml" +# - "./opensearch-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./datanode-security-config.yml:/usr/share/opensearch/config/opensearch-security/config.yml" + - "./keystore.jks:/usr/share/opensearch/config/keystore.jks" + + datanode1: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch1 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8999:8999/tcp" # DataNode API + - "9200:9200" + volumes: + - "graylog-datanode-01:/var/lib/graylog-datanode" + - "opensearch-data-01:/var/lib/graylog-datanode/opensearch/data" + + datanode2: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch2 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8998:8999/tcp" # DataNode API + volumes: + - "graylog-datanode-02:/var/lib/graylog-datanode" + - "opensearch-data-02:/var/lib/graylog-datanode/opensearch/data" + + datanode3: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2.0}" + hostname: opensearch3 + depends_on: + - "mongodb" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8997:8999/tcp" # DataNode API + volumes: + - "graylog-datanode-03:/var/lib/graylog-datanode" + - "opensearch-data-03:/var/lib/graylog-datanode/opensearch/data" + + graylog1: + image: "${GRAYLOG_IMAGE:-graylog/graylog:5.2.0}" + hostname: "graylog1" + depends_on: + - "mongodb" + entrypoint: "/docker-entrypoint.sh" + environment: + GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id" + GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}" + GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}" + GRAYLOG_HTTP_EXTERNAL_URI: "http://127.0.0.1:9000/" + GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog" + GRAYLOG_REPORT_DISABLE_SANDBOX: "true" +# GRAYLOG_CA_KEYSTORE_FILE: "/usr/share/graylog/data/keystore.jks" +# GRAYLOG_CA_PASSWORD: "password" +# GRAYLOG_ELASTICSEARCH_HOSTS: "https://admin:admin@opensearch1:9200,https://admin:admin@opensearch2:9200,https://admin:admin@opensearch3:9200" + GRAYLOG_ENABLE_PREFLIGHT_WEB: "true" + ports: + - "9000:9000" # Graylog web interface and REST API + - "1514:1514" # Syslog TCP + - "1514:1514/udp" # Syslog UDP + - "12201:12201" # GELF TCP + - "12201:12201/udp" # GELF UDP + volumes: + - "graylog-data-01:/usr/share/graylog/data/data" + - "graylog-journal-01:/usr/share/graylog/data/journal" + - "./keystore.jks:/usr/share/graylog/data/keystore.jks" + +volumes: + graylog-datanode-01: + graylog-datanode-02: + graylog-datanode-03: + graylog-data-01: + graylog-journal-01: + mongodb-data: + opensearch-data-01: + opensearch-data-02: + opensearch-data-03: diff --git a/data-node/migration/opensearch-security-config.yml b/data-node/migration/opensearch-security-config.yml new file mode 100644 index 0000000000..1493a0d7f1 --- /dev/null +++ b/data-node/migration/opensearch-security-config.yml @@ -0,0 +1,250 @@ +--- + +# This is the main OpenSearch Security configuration file where authentication +# and authorization is defined. +# +# You need to configure at least one authentication domain in the authc of this file. +# An authentication domain is responsible for extracting the user credentials from +# the request and for validating them against an authentication backend like Active Directory for example. +# +# If more than one authentication domain is configured the first one which succeeds wins. +# If all authentication domains fail then the request is unauthenticated. +# In this case an exception is thrown and/or the HTTP status is set to 401. +# +# After authentication authorization (authz) will be applied. There can be zero or more authorizers which collect +# the roles from a given backend for the authenticated user. +# +# Both, authc and auth can be enabled/disabled separately for REST and TRANSPORT layer. Default is true for both. +# http_enabled: true +# transport_enabled: true +# +# For HTTP it is possible to allow anonymous authentication. If that is the case then the HTTP authenticators try to +# find user credentials in the HTTP request. If credentials are found then the user gets regularly authenticated. +# If none can be found the user will be authenticated as an "anonymous" user. This user has always the username "anonymous" +# and one role named "anonymous_backendrole". +# If you enable anonymous authentication all HTTP authenticators will not challenge. +# +# +# Note: If you define more than one HTTP authenticators make sure to put non-challenging authenticators like "proxy" or "clientcert" +# first and the challenging one last. +# Because it's not possible to challenge a client with two different authentication methods (for example +# Kerberos and Basic) only one can have the challenge flag set to true. You can cope with this situation +# by using pre-authentication, e.g. sending a HTTP Basic authentication header in the request. +# +# Default value of the challenge flag is true. +# +# +# HTTP +# basic (challenging) +# proxy (not challenging, needs xff) +# kerberos (challenging) +# clientcert (not challenging, needs https) +# jwt (not challenging) +# host (not challenging) #DEPRECATED, will be removed in a future version. +# host based authentication is configurable in roles_mapping + +# Authc +# internal +# noop +# ldap + +# Authz +# ldap +# noop + + + +_meta: + type: "config" + config_version: 2 + +config: + dynamic: + # Set filtered_alias_mode to 'disallow' to forbid more than 2 filtered aliases per index + # Set filtered_alias_mode to 'warn' to allow more than 2 filtered aliases per index but warns about it (default) + # Set filtered_alias_mode to 'nowarn' to allow more than 2 filtered aliases per index silently + #filtered_alias_mode: warn + #do_not_fail_on_forbidden: false + #kibana: + # Kibana multitenancy + #multitenancy_enabled: true + #private_tenant_enabled: true + #default_tenant: "" + #server_username: kibanaserver + #index: '.kibana' + http: + anonymous_auth_enabled: false + xff: + enabled: false + internalProxies: '192\.168\.0\.10|192\.168\.0\.11' # regex pattern + #internalProxies: '.*' # trust all internal proxies, regex pattern + #remoteIpHeader: 'x-forwarded-for' + ###### see https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html for regex help + ###### more information about XFF https://en.wikipedia.org/wiki/X-Forwarded-For + ###### and here https://tools.ietf.org/html/rfc7239 + ###### and https://tomcat.apache.org/tomcat-8.0-doc/config/valve.html#Remote_IP_Valve + authc: + kerberos_auth_domain: + http_enabled: false + transport_enabled: false + order: 6 + http_authenticator: + type: kerberos + challenge: true + config: + # If true a lot of kerberos/security related debugging output will be logged to standard out + krb_debug: false + # If true then the realm will be stripped from the user name + strip_realm_from_principal: true + authentication_backend: + type: noop + basic_internal_auth_domain: + description: "Authenticate via HTTP Basic against internal users database" + http_enabled: true + transport_enabled: true + order: 4 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + proxy_auth_domain: + description: "Authenticate via proxy" + http_enabled: false + transport_enabled: false + order: 3 + http_authenticator: + type: proxy + challenge: false + config: + user_header: "x-proxy-user" + roles_header: "x-proxy-roles" + authentication_backend: + type: noop + jwt_auth_domain: + description: "Authenticate via Json Web Token" + http_enabled: false + transport_enabled: false + order: 0 + http_authenticator: + type: jwt + challenge: false + config: + signing_key: "base64 encoded HMAC key or public RSA/ECDSA pem key" + jwt_header: "Authorization" + jwt_url_parameter: null + jwt_clock_skew_tolerance_seconds: 30 + roles_key: null + subject_key: null + authentication_backend: + type: noop + clientcert_auth_domain: + description: "Authenticate via SSL client certificates" + http_enabled: false + transport_enabled: false + order: 2 + http_authenticator: + type: clientcert + config: + username_attribute: cn #optional, if omitted DN becomes username + challenge: false + authentication_backend: + type: noop + ldap: + description: "Authenticate via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + order: 5 + http_authenticator: + type: basic + challenge: false + authentication_backend: + # LDAP authentication backend (authenticate users against a LDAP or Active Directory) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(sAMAccountName={0})' + # Use this attribute from the user as username (if not set then DN is used) + username_attribute: null + authz: + roles_from_myldap: + description: "Authorize via LDAP or Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + # LDAP authorization backend (gather roles from a LDAP or Active Directory, you have to configure the above LDAP authentication backend settings too) + type: ldap + config: + # enable ldaps + enable_ssl: false + # enable start tls, enable_ssl should be false + enable_start_tls: false + # send client certificate + enable_ssl_client_auth: false + # verify ldap hostname + verify_hostnames: true + hosts: + - localhost:8389 + bind_dn: null + password: null + rolebase: 'ou=groups,dc=example,dc=com' + # Filter to search for roles (currently in the whole subtree beneath rolebase) + # {0} is substituted with the DN of the user + # {1} is substituted with the username + # {2} is substituted with an attribute value from user's directory entry, of the authenticated user. Use userroleattribute to specify the name of the attribute + rolesearch: '(member={0})' + # Specify the name of the attribute which value should be substituted with {2} above + userroleattribute: null + # Roles as an attribute of the user entry + userrolename: disabled + #userrolename: memberOf + # The attribute in a role entry containing the name of that role, Default is "name". + # Can also be "dn" to use the full DN as rolename. + rolename: cn + # Resolve nested roles transitive (roles which are members of other roles and so on ...) + resolve_nested_roles: true + userbase: 'ou=people,dc=example,dc=com' + # Filter to search for users (currently in the whole subtree beneath userbase) + # {0} is substituted with the username + usersearch: '(uid={0})' + # Skip users matching a user name, a wildcard or a regex pattern + #skip_users: + # - 'cn=Michael Jackson,ou*people,o=TEST' + # - '/\S*/' + roles_from_another_ldap: + description: "Authorize via another Active Directory" + http_enabled: false + transport_enabled: false + authorization_backend: + type: ldap + #config goes here ... + # auth_failure_listeners: + # ip_rate_limiting: + # type: ip + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000 + # internal_authentication_backend_limiting: + # type: username + # authentication_backend: intern + # allowed_tries: 10 + # time_window_seconds: 3600 + # block_expiry_seconds: 600 + # max_blocked_clients: 100000 + # max_tracked_clients: 100000