feat(global-search): dashboard globalsearch apis (#3831)

Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com>

config/config.example.toml

@@ -585,3 +585,17 @@ encryption_manager = "aws_kms"            # Encryption manager client to be used
 [encryption_management.aws_kms]
 key_id = "kms_key_id" # The AWS key ID used by the KMS SDK for decrypting data.
 region = "kms_region" # The AWS region used by the KMS SDK for decrypting data.
+
+[opensearch]
+host = "https://localhost:9200"
+
+[opensearch.auth]
+auth = "basic"
+username = "admin"
+password = "admin"
+region = "eu-central-1"
+
+[opensearch.indexes]
+payment_attempts = "hyperswitch-payment-attempt-events"
+payment_intents = "hyperswitch-payment-intent-events"
+refunds = "hyperswitch-refund-events"

config/deployments/env_specific.toml

@@ -42,7 +42,11 @@ client_secret = "paypal_client_secret"
 partner_id = "paypal_partner_id"
 
 [connector_request_reference_id_config]
-merchant_ids_send_payment_id_as_connector_request_id = ["merchant_id_1", "merchant_id_2", "etc.,"]
+merchant_ids_send_payment_id_as_connector_request_id = [
+    "merchant_id_1",
+    "merchant_id_2",
+    "etc.,",
+]
 
 [cors]
 max_age = 30                            # Maximum time (in seconds) for which this CORS request may be cached.
@@ -174,7 +178,10 @@ default_command_timeout = 30  # An optional timeout to apply to all commands. In
 unresponsive_timeout = 10 # An optional timeout for Unresponsive commands in seconds. This should be less than default_command_timeout.
 max_feed_count = 200 # The maximum number of frames that will be fed to a socket before flushing.
 cluster_enabled = true # boolean
-cluster_urls = ["redis.cluster.uri-1:8080", "redis.cluster.uri-2:4115"] # List of redis cluster urls
+cluster_urls = [
+    "redis.cluster.uri-1:8080",
+    "redis.cluster.uri-2:4115",
+] # List of redis cluster urls
 
 # Replica SQL data store credentials
 [replica_database]
@@ -193,6 +200,20 @@ payment_function = "report_download_config_payment_function" # Config to downloa
 refund_function = "report_download_config_refund_function"   # Config to download refund report
 region = "report_download_config_region"                     # Region of the bucket
 
+[opensearch]
+host = "https://localhost:9200"
+
+[opensearch.auth]
+auth = "basic"
+username = "admin"
+password = "admin"
+region = "eu-central-1"
+
+[opensearch.indexes]
+payment_attempts = "hyperswitch-payment-attempt-events"
+payment_intents = "hyperswitch-payment-intent-events"
+refunds = "hyperswitch-refund-events"
+
 # This section provides some secret values.
 [secrets]
 master_enc_key = "sample_key"            # Master Encryption key used to encrypt merchant wise encryption key. Should be 32-byte long.

config/development.toml

@@ -582,3 +582,17 @@ file_storage_backend = "file_system"
 
 [unmasked_headers]
 keys = "user-agent"
+
+[opensearch]
+host = "https://localhost:9200"
+
+[opensearch.auth]
+auth = "basic"
+username = "admin"
+password = "admin"
+region = "eu-central-1"
+
+[opensearch.indexes]
+payment_attempts = "hyperswitch-payment-attempt-events"
+payment_intents = "hyperswitch-payment-intent-events"
+refunds = "hyperswitch-refund-events"

config/docker_compose.toml

@@ -441,3 +441,17 @@ file_storage_backend = "file_system"
 
 [unmasked_headers]
 keys = "user-agent"
+
+[opensearch]
+host = "https://opensearch:9200"
+
+[opensearch.auth]
+auth = "basic"
+username = "admin"
+password = "admin"
+region = "eu-central-1"
+
+[opensearch.indexes]
+payment_attempts = "hyperswitch-payment-attempt-events"
+payment_intents = "hyperswitch-payment-intent-events"
+refunds = "hyperswitch-refund-events"

crates/analytics/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "analytics"
 version = "0.1.0"
-description = "Analytics / Reports related functionality"
+description = "Analytics / Reports / Search related functionality"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,29 +9,43 @@ edition = "2021"
 
 [dependencies]
 # First party crates
-api_models = { version = "0.1.0", path = "../api_models" , features = ["errors"]}
+api_models = { version = "0.1.0", path = "../api_models", features = [
+    "errors",
+] }
 storage_impl = { version = "0.1.0", path = "../storage_impl", default-features = false }
 common_utils = { version = "0.1.0", path = "../common_utils" }
 external_services = { version = "0.1.0", path = "../external_services", default-features = false }
 hyperswitch_interfaces = { version = "0.1.0", path = "../hyperswitch_interfaces" }
 masking = { version = "0.1.0", path = "../masking" }
-router_env = { version = "0.1.0", path = "../router_env", features = ["log_extra_implicit_fields", "log_custom_entries_to_extra"] }
+router_env = { version = "0.1.0", path = "../router_env", features = [
-diesel_models = { version = "0.1.0", path = "../diesel_models", features = ["kv_store"] }
+    "log_extra_implicit_fields",
+    "log_custom_entries_to_extra",
+] }
+diesel_models = { version = "0.1.0", path = "../diesel_models", features = [
+    "kv_store",
+] }
 
 #Third Party dependencies
 actix-web = "4.3.1"
 async-trait = "0.1.68"
-aws-config = { version = "0.55.3" }
+aws-config = { version = "1.1.6", features = ["behavior-version-latest"] }
-aws-sdk-lambda = { version = "0.28.0" }
+aws-sdk-lambda = { version = "1.1.4" }
-aws-smithy-types = { version = "0.55.3" }
+aws-smithy-types = { version = "1.1.6" }
 bigdecimal = { version = "0.3.1", features = ["serde"] }
 error-stack = "0.3.1"
 futures = "0.3.28"
+opensearch = { version = "2.2.0", features = ["aws-auth"] }
 once_cell = "1.18.0"
 reqwest = { version = "0.11.18", features = ["serde_json"] }
 serde = { version = "1.0.193", features = ["derive", "rc"] }
 serde_json = "1.0.108"
-sqlx = { version = "0.6.3", features = ["postgres", "runtime-actix", "runtime-actix-native-tls", "time", "bigdecimal"] }
+sqlx = { version = "0.6.3", features = [
+    "postgres",
+    "runtime-actix",
+    "runtime-actix-native-tls",
+    "time",
+    "bigdecimal",
+] }
 strum = { version = "0.25.0", features = ["derive"] }
 thiserror = "1.0.43"
 time = { version = "0.3.21", features = ["serde", "serde-well-known", "std"] }

crates/analytics/src/lambda_utils.rs

@@ -1,5 +1,5 @@
-use aws_config::{self, meta::region::RegionProviderChain};
+use aws_config::{self, meta::region::RegionProviderChain, Region};
-use aws_sdk_lambda::{config::Region, types::InvocationType::Event, Client};
+use aws_sdk_lambda::{types::InvocationType::Event, Client};
 use aws_smithy_types::Blob;
 use common_utils::errors::CustomResult;
 use error_stack::{IntoReport, ResultExt};

crates/analytics/src/lib.rs

@@ -12,6 +12,7 @@ pub mod connector_events;
 pub mod health_check;
 pub mod outgoing_webhook_event;
 pub mod sdk_events;
+pub mod search;
 mod sqlx;
 mod types;
 use api_event::metrics::{ApiEventMetric, ApiEventMetricRow};
@@ -664,3 +665,42 @@ pub struct ReportConfig {
     pub dispute_function: String,
     pub region: String,
 }
+
+#[derive(Clone, Debug, serde::Deserialize)]
+#[serde(tag = "auth")]
+#[serde(rename_all = "lowercase")]
+pub enum OpensearchAuth {
+    Basic { username: String, password: String },
+    Aws { region: String },
+}
+
+#[derive(Clone, Debug, serde::Deserialize)]
+pub struct OpensearchIndexes {
+    pub payment_attempts: String,
+    pub payment_intents: String,
+    pub refunds: String,
+}
+
+#[derive(Clone, Debug, serde::Deserialize)]
+pub struct OpensearchConfig {
+    host: String,
+    auth: OpensearchAuth,
+    indexes: OpensearchIndexes,
+}
+
+impl Default for OpensearchConfig {
+    fn default() -> Self {
+        Self {
+            host: "https://localhost:9200".to_string(),
+            auth: OpensearchAuth::Basic {
+                username: "admin".to_string(),
+                password: "admin".to_string(),
+            },
+            indexes: OpensearchIndexes {
+                payment_attempts: "hyperswitch-payment-attempt-events".to_string(),
+                payment_intents: "hyperswitch-payment-intent-events".to_string(),
+                refunds: "hyperswitch-refund-events".to_string(),
+            },
+        }
+    }
+}

crates/analytics/src/search.rs

@@ -0,0 +1,150 @@
+use api_models::analytics::search::{
+    GetGlobalSearchRequest, GetSearchRequestWithIndex, GetSearchResponse, OpenMsearchOutput,
+    OpensearchOutput, SearchIndex,
+};
+use aws_config::{self, meta::region::RegionProviderChain, Region};
+use common_utils::errors::CustomResult;
+use opensearch::{
+    auth::Credentials,
+    cert::CertificateValidation,
+    http::{
+        request::JsonBody,
+        transport::{SingleNodeConnectionPool, TransportBuilder},
+        Url,
+    },
+    MsearchParts, OpenSearch, SearchParts,
+};
+use serde_json::{json, Value};
+use strum::IntoEnumIterator;
+
+use crate::{errors::AnalyticsError, OpensearchAuth, OpensearchConfig, OpensearchIndexes};
+
+#[derive(Debug, thiserror::Error)]
+pub enum OpensearchError {
+    #[error("Opensearch connection error")]
+    ConnectionError,
+    #[error("Opensearch NON-200 response content: '{0}'")]
+    ResponseNotOK(String),
+    #[error("Opensearch response error")]
+    ResponseError,
+}
+
+pub fn search_index_to_opensearch_index(index: SearchIndex, config: &OpensearchIndexes) -> String {
+    match index {
+        SearchIndex::PaymentAttempts => config.payment_attempts.clone(),
+        SearchIndex::PaymentIntents => config.payment_intents.clone(),
+        SearchIndex::Refunds => config.refunds.clone(),
+    }
+}
+
+async fn get_opensearch_client(config: OpensearchConfig) -> Result<OpenSearch, OpensearchError> {
+    let url = Url::parse(&config.host).map_err(|_| OpensearchError::ConnectionError)?;
+    let transport = match config.auth {
+        OpensearchAuth::Basic { username, password } => {
+            let credentials = Credentials::Basic(username, password);
+            TransportBuilder::new(SingleNodeConnectionPool::new(url))
+                .cert_validation(CertificateValidation::None)
+                .auth(credentials)
+                .build()
+                .map_err(|_| OpensearchError::ConnectionError)?
+        }
+        OpensearchAuth::Aws { region } => {
+            let region_provider = RegionProviderChain::first_try(Region::new(region));
+            let sdk_config = aws_config::from_env().region(region_provider).load().await;
+            let conn_pool = SingleNodeConnectionPool::new(url);
+            TransportBuilder::new(conn_pool)
+                .auth(
+                    sdk_config
+                        .clone()
+                        .try_into()
+                        .map_err(|_| OpensearchError::ConnectionError)?,
+                )
+                .service_name("es")
+                .build()
+                .map_err(|_| OpensearchError::ConnectionError)?
+        }
+    };
+    Ok(OpenSearch::new(transport))
+}
+
+pub async fn msearch_results(
+    req: GetGlobalSearchRequest,
+    merchant_id: &String,
+    config: OpensearchConfig,
+) -> CustomResult<Vec<GetSearchResponse>, AnalyticsError> {
+    let client = get_opensearch_client(config.clone())
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    let mut msearch_vector: Vec<JsonBody<Value>> = vec![];
+    for index in SearchIndex::iter() {
+        msearch_vector
+            .push(json!({"index": search_index_to_opensearch_index(index,&config.indexes)}).into());
+        msearch_vector.push(json!({"query": {"bool": {"must": {"query_string": {"query": req.query}}, "filter": {"match_phrase": {"merchant_id": merchant_id}}}}}).into());
+    }
+
+    let response = client
+        .msearch(MsearchParts::None)
+        .body(msearch_vector)
+        .send()
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    let response_body = response
+        .json::<OpenMsearchOutput<Value>>()
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    Ok(response_body
+        .responses
+        .into_iter()
+        .zip(SearchIndex::iter())
+        .map(|(index_hit, index)| GetSearchResponse {
+            count: index_hit.hits.total.value,
+            index,
+            hits: index_hit
+                .hits
+                .hits
+                .into_iter()
+                .map(|hit| hit._source)
+                .collect(),
+        })
+        .collect())
+}
+
+pub async fn search_results(
+    req: GetSearchRequestWithIndex,
+    merchant_id: &String,
+    config: OpensearchConfig,
+) -> CustomResult<GetSearchResponse, AnalyticsError> {
+    let search_req = req.search_req;
+
+    let client = get_opensearch_client(config.clone())
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    let response = client
+        .search(SearchParts::Index(&[&search_index_to_opensearch_index(req.index.clone(),&config.indexes)]))
+        .from(search_req.offset)
+        .size(search_req.count)
+        .body(json!({"query": {"bool": {"must": {"query_string": {"query": search_req.query}}, "filter": {"match_phrase": {"merchant_id": merchant_id}}}}}))
+        .send()
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    let response_body = response
+        .json::<OpensearchOutput<Value>>()
+        .await
+        .map_err(|_| AnalyticsError::UnknownError)?;
+
+    Ok(GetSearchResponse {
+        count: response_body.hits.total.value,
+        index: req.index,
+        hits: response_body
+            .hits
+            .hits
+            .into_iter()
+            .map(|hit| hit._source)
+            .collect(),
+    })
+}

crates/api_models/src/analytics.rs

@@ -19,6 +19,7 @@ pub mod outgoing_webhook_event;
 pub mod payments;
 pub mod refunds;
 pub mod sdk_events;
+pub mod search;
 
 #[derive(Debug, serde::Serialize)]
 pub struct NameDescription {
@@ -251,7 +252,6 @@ pub struct GetApiEventMetricRequest {
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 #[serde(rename_all = "camelCase")]
-
 pub struct GetDisputeFilterRequest {
     pub time_range: TimeRange,
     #[serde(default)]

crates/api_models/src/analytics/search.rs

@@ -0,0 +1,73 @@
+use serde_json::Value;
+
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
+pub struct SearchFilters {
+    pub payment_method: Option<Vec<String>>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetGlobalSearchRequest {
+    pub query: String,
+    #[serde(default)]
+    pub filters: Option<SearchFilters>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetSearchRequest {
+    pub offset: i64,
+    pub count: i64,
+    pub query: String,
+    #[serde(default)]
+    pub filters: Option<SearchFilters>,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetSearchRequestWithIndex {
+    pub index: SearchIndex,
+    pub search_req: GetSearchRequest,
+}
+
+#[derive(Debug, strum::EnumIter, Clone, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SearchIndex {
+    PaymentAttempts,
+    PaymentIntents,
+    Refunds,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GetSearchResponse {
+    pub count: u64,
+    pub index: SearchIndex,
+    pub hits: Vec<Value>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpenMsearchOutput<T> {
+    pub responses: Vec<OpensearchOutput<T>>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchOutput<T> {
+    pub hits: OpensearchResults<T>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchResults<T> {
+    pub total: OpensearchResultsTotal,
+    pub hits: Vec<OpensearchHits<T>>,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchResultsTotal {
+    pub value: u64,
+}
+
+#[derive(Debug, serde::Deserialize)]
+pub struct OpensearchHits<T> {
+    pub _source: T,
+}

crates/api_models/src/events.rs

@@ -23,7 +23,7 @@ use crate::{
     admin::*,
     analytics::{
         api_event::*, connector_events::ConnectorEventsRequest,
-        outgoing_webhook_event::OutgoingWebhookLogsRequest, sdk_events::*, *,
+        outgoing_webhook_event::OutgoingWebhookLogsRequest, sdk_events::*, search::*, *,
     },
     api_keys::*,
     cards_info::*,
@@ -96,6 +96,10 @@ impl_misc_api_event_type!(
     ReportRequest,
     ConnectorEventsRequest,
     OutgoingWebhookLogsRequest,
+    GetGlobalSearchRequest,
+    GetSearchRequest,
+    GetSearchResponse,
+    GetSearchRequestWithIndex,
     GetDisputeFilterRequest,
     DisputeFiltersResponse,
     GetDisputeMetricRequest

crates/router/src/analytics.rs

@@ -8,6 +8,9 @@ pub mod routes {
         outgoing_webhook_event::outgoing_webhook_events_core, sdk_events::sdk_events_core,
     };
     use api_models::analytics::{
+        search::{
+            GetGlobalSearchRequest, GetSearchRequest, GetSearchRequestWithIndex, SearchIndex,
+        },
         GenerateReportRequest, GetApiEventFiltersRequest, GetApiEventMetricRequest,
         GetDisputeMetricRequest, GetPaymentFiltersRequest, GetPaymentMetricRequest,
         GetRefundFilterRequest, GetRefundMetricRequest, GetSdkEventFiltersRequest,
@@ -89,6 +92,12 @@ pub mod routes {
                         web::resource("metrics/api_events")
                             .route(web::post().to(get_api_events_metrics)),
                     )
+                    .service(
+                        web::resource("search").route(web::post().to(get_global_search_results)),
+                    )
+                    .service(
+                        web::resource("search/{domain}").route(web::post().to(get_search_results)),
+                    )
                     .service(
                         web::resource("filters/disputes")
                             .route(web::post().to(get_dispute_filters)),
@@ -113,7 +122,7 @@ pub mod routes {
             state,
             &req,
             domain.into_inner(),
-            |_, _, domain| async {
+            |_, _, domain: analytics::AnalyticsDomain| async {
                 analytics::core::get_domain_info(domain)
                     .await
                     .map(ApplicationResponse::Json)
@@ -592,6 +601,63 @@ pub mod routes {
         .await
     }
 
+    pub async fn get_global_search_results(
+        state: web::Data<AppState>,
+        req: actix_web::HttpRequest,
+        json_payload: web::Json<GetGlobalSearchRequest>,
+    ) -> impl Responder {
+        let flow = AnalyticsFlow::GetGlobalSearchResults;
+        Box::pin(api::server_wrap(
+            flow,
+            state.clone(),
+            &req,
+            json_payload.into_inner(),
+            |state, auth: AuthenticationData, req| async move {
+                analytics::search::msearch_results(
+                    req,
+                    &auth.merchant_account.merchant_id,
+                    state.conf.opensearch.clone(),
+                )
+                .await
+                .map(ApplicationResponse::Json)
+            },
+            &auth::JWTAuth(Permission::Analytics),
+            api_locking::LockAction::NotApplicable,
+        ))
+        .await
+    }
+
+    pub async fn get_search_results(
+        state: web::Data<AppState>,
+        req: actix_web::HttpRequest,
+        json_payload: web::Json<GetSearchRequest>,
+        index: actix_web::web::Path<SearchIndex>,
+    ) -> impl Responder {
+        let flow = AnalyticsFlow::GetSearchResults;
+        let indexed_req = GetSearchRequestWithIndex {
+            search_req: json_payload.into_inner(),
+            index: index.into_inner(),
+        };
+        Box::pin(api::server_wrap(
+            flow,
+            state.clone(),
+            &req,
+            indexed_req,
+            |state, auth: AuthenticationData, req| async move {
+                analytics::search::search_results(
+                    req,
+                    &auth.merchant_account.merchant_id,
+                    state.conf.opensearch.clone(),
+                )
+                .await
+                .map(ApplicationResponse::Json)
+            },
+            &auth::JWTAuth(Permission::Analytics),
+            api_locking::LockAction::NotApplicable,
+        ))
+        .await
+    }
+
     pub async fn get_dispute_filters(
         state: web::Data<AppState>,
         req: actix_web::HttpRequest,

crates/router/src/configs/secrets_transformers.rs

@@ -350,6 +350,8 @@ pub(crate) async fn fetch_raw_secrets(
         payment_link: conf.payment_link,
         #[cfg(feature = "olap")]
         analytics,
+        #[cfg(feature = "olap")]
+        opensearch: conf.opensearch,
         #[cfg(feature = "kv_store")]
         kv_config: conf.kv_config,
         #[cfg(feature = "frm")]

crates/router/src/configs/settings.rs

@@ -4,7 +4,7 @@ use std::{
 };
 
 #[cfg(feature = "olap")]
-use analytics::ReportConfig;
+use analytics::{OpensearchConfig, ReportConfig};
 use api_models::{enums, payment_methods::RequiredFieldInfo};
 use common_utils::ext_traits::ConfigExt;
 use config::{Environment, File};
@@ -112,6 +112,8 @@ pub struct Settings<S: SecretState> {
     pub frm: Frm,
     #[cfg(feature = "olap")]
     pub report_download_config: ReportConfig,
+    #[cfg(feature = "olap")]
+    pub opensearch: OpensearchConfig,
     pub events: EventsConfig,
     #[cfg(feature = "olap")]
     pub connector_onboarding: SecretStateContainer<ConnectorOnboarding, S>,

crates/router_env/src/lib.rs

@@ -54,6 +54,8 @@ pub enum AnalyticsFlow {
     GetApiEventFilters,
     GetConnectorEvents,
     GetOutgoingWebhookEvents,
+    GetGlobalSearchResults,
+    GetSearchResults,
     GetDisputeFilters,
     GetDisputeMetrics,
 }