rust/hyperswitch
1
0
Fork 0
mirror of https://github.com/juspay/hyperswitch.git synced 2025-11-01 11:06:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(opensearch): refactoring (#4244)

Browse Source 
Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com>
Co-authored-by: Sanchith Hegde <22217505+SanchithHegde@users.noreply.github.com>
This commit is contained in:
ivor-juspay
2024-05-02 15:50:58 +05:30
committed by GitHub
parent e4ed1e6395
commit 22cb01ac1e
16 changed files with 693 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
crates/analytics/src/lib.rs
View File

@ -10,6 +10,7 @@ pub mod refunds;
pub mod api_event;
pub mod connector_events;
pub mod health_check;
pub mod opensearch;
pub mod outgoing_webhook_event;
pub mod sdk_events;
pub mod search;
@ -668,47 +669,6 @@ pub struct ReportConfig {
pub region: String,
}
#[derive(Clone, Debug, serde::Deserialize)]
#[serde(tag = "auth")]
#[serde(rename_all = "lowercase")]
pub enum OpensearchAuth {
Basic { username: String, password: String },
Aws { region: String },
}
#[derive(Clone, Debug, serde::Deserialize)]
pub struct OpensearchIndexes {
pub payment_attempts: String,
pub payment_intents: String,
pub refunds: String,
pub disputes: String,
}
#[derive(Clone, Debug, serde::Deserialize)]
pub struct OpensearchConfig {
host: String,
auth: OpensearchAuth,
indexes: OpensearchIndexes,
}
impl Default for OpensearchConfig {
fn default() -> Self {
Self {
host: "https://localhost:9200".to_string(),
auth: OpensearchAuth::Basic {
username: "admin".to_string(),
password: "admin".to_string(),
},
indexes: OpensearchIndexes {
payment_attempts: "hyperswitch-payment-attempt-events".to_string(),
payment_intents: "hyperswitch-payment-intent-events".to_string(),
refunds: "hyperswitch-refund-events".to_string(),
disputes: "hyperswitch-dispute-events".to_string(),
},
}
}
}
/// Analytics Flow routes Enums
/// Info - Dimensions and filters available for the domain
/// Filters - Set of values present for the dimension

397
crates/analytics/src/opensearch.rs Normal file
View File

@ -0,0 +1,397 @@
use api_models::{
analytics::search::SearchIndex,
errors::types::{ApiError, ApiErrorResponse},
};
use aws_config::{self, meta::region::RegionProviderChain, Region};
use common_utils::errors::{CustomResult, ErrorSwitch};
use data_models::errors::{StorageError, StorageResult};
use error_stack::ResultExt;
use opensearch::{
auth::Credentials,
cert::CertificateValidation,
cluster::{Cluster, ClusterHealthParts},
http::{
request::JsonBody,
response::Response,
transport::{SingleNodeConnectionPool, Transport, TransportBuilder},
Url,
},
MsearchParts, OpenSearch, SearchParts,
};
use serde_json::{json, Value};
use storage_impl::errors::ApplicationError;
use strum::IntoEnumIterator;
use super::{health_check::HealthCheck, query::QueryResult, types::QueryExecutionError};
use crate::query::QueryBuildingError;
#[derive(Clone, Debug, serde::Deserialize)]
#[serde(tag = "auth")]
#[serde(rename_all = "lowercase")]
pub enum OpenSearchAuth {
Basic { username: String, password: String },
Aws { region: String },
}
#[derive(Clone, Debug, serde::Deserialize)]
pub struct OpenSearchIndexes {
pub payment_attempts: String,
pub payment_intents: String,
pub refunds: String,
pub disputes: String,
}
#[derive(Clone, Debug, serde::Deserialize)]
pub struct OpenSearchConfig {
host: String,
auth: OpenSearchAuth,
indexes: OpenSearchIndexes,
}
impl Default for OpenSearchConfig {
fn default() -> Self {
Self {
host: "https://localhost:9200".to_string(),
auth: OpenSearchAuth::Basic {
username: "admin".to_string(),
password: "admin".to_string(),
},
indexes: OpenSearchIndexes {
payment_attempts: "hyperswitch-payment-attempt-events".to_string(),
payment_intents: "hyperswitch-payment-intent-events".to_string(),
refunds: "hyperswitch-refund-events".to_string(),
disputes: "hyperswitch-dispute-events".to_string(),
},
}
}
}
#[derive(Debug, thiserror::Error)]
pub enum OpenSearchError {
#[error("Opensearch connection error")]
ConnectionError,
#[error("Opensearch NON-200 response content: '{0}'")]
ResponseNotOK(String),
#[error("Opensearch response error")]
ResponseError,
#[error("Opensearch query building error")]
QueryBuildingError,
}
impl ErrorSwitch<OpenSearchError> for QueryBuildingError {
fn switch(&self) -> OpenSearchError {
OpenSearchError::QueryBuildingError
}
}
impl ErrorSwitch<ApiErrorResponse> for OpenSearchError {
fn switch(&self) -> ApiErrorResponse {
match self {
Self::ConnectionError => ApiErrorResponse::InternalServerError(ApiError::new(
"IR",
0,
"Connection error",
None,
)),
Self::ResponseNotOK(response) => ApiErrorResponse::InternalServerError(ApiError::new(
"IR",
0,
format!("Something went wrong {}", response),
None,
)),
Self::ResponseError => ApiErrorResponse::InternalServerError(ApiError::new(
"IR",
0,
"Something went wrong",
None,
)),
Self::QueryBuildingError => ApiErrorResponse::InternalServerError(ApiError::new(
"IR",
0,
"Query building error",
None,
)),
}
}
}
#[derive(Clone, Debug)]
pub struct OpenSearchClient {
pub client: OpenSearch,
pub transport: Transport,
pub indexes: OpenSearchIndexes,
}
impl OpenSearchClient {
pub async fn create(conf: &OpenSearchConfig) -> CustomResult<Self, OpenSearchError> {
let url = Url::parse(&conf.host).map_err(|_| OpenSearchError::ConnectionError)?;
let transport = match &conf.auth {
OpenSearchAuth::Basic { username, password } => {
let credentials = Credentials::Basic(username.clone(), password.clone());
TransportBuilder::new(SingleNodeConnectionPool::new(url))
.cert_validation(CertificateValidation::None)
.auth(credentials)
.build()
.map_err(|_| OpenSearchError::ConnectionError)?
}
OpenSearchAuth::Aws { region } => {
let region_provider = RegionProviderChain::first_try(Region::new(region.clone()));
let sdk_config = aws_config::from_env().region(region_provider).load().await;
let conn_pool = SingleNodeConnectionPool::new(url);
TransportBuilder::new(conn_pool)
.auth(
sdk_config
.clone()
.try_into()
.map_err(|_| OpenSearchError::ConnectionError)?,
)
.service_name("es")
.build()
.map_err(|_| OpenSearchError::ConnectionError)?
}
};
Ok(Self {
transport: transport.clone(),
client: OpenSearch::new(transport),
indexes: conf.indexes.clone(),
})
}
pub fn search_index_to_opensearch_index(&self, index: SearchIndex) -> String {
match index {
SearchIndex::PaymentAttempts => self.indexes.payment_attempts.clone(),
SearchIndex::PaymentIntents => self.indexes.payment_intents.clone(),
SearchIndex::Refunds => self.indexes.refunds.clone(),
SearchIndex::Disputes => self.indexes.disputes.clone(),
}
}
pub async fn execute(
&self,
query_builder: OpenSearchQueryBuilder,
) -> CustomResult<Response, OpenSearchError> {
match query_builder.query_type {
OpenSearchQuery::Msearch => {
let search_indexes = SearchIndex::iter();
let payload = query_builder
.construct_payload(search_indexes.clone().collect())
.change_context(OpenSearchError::QueryBuildingError)?;
let payload_with_indexes = payload.into_iter().zip(search_indexes).fold(
Vec::new(),
|mut payload_with_indexes, (index_hit, index)| {
payload_with_indexes.push(
json!({"index": self.search_index_to_opensearch_index(index)}).into(),
);
payload_with_indexes.push(JsonBody::new(index_hit.clone()));
payload_with_indexes
},
);
self.client
.msearch(MsearchParts::None)
.body(payload_with_indexes)
.send()
.await
.change_context(OpenSearchError::ResponseError)
}
OpenSearchQuery::Search(index) => {
let payload = query_builder
.clone()
.construct_payload(vec![index])
.change_context(OpenSearchError::QueryBuildingError)?;
let final_payload = payload.first().unwrap_or(&Value::Null);
self.client
.search(SearchParts::Index(&[
&self.search_index_to_opensearch_index(index)
]))
.from(query_builder.offset.unwrap_or(0))
.size(query_builder.count.unwrap_or(10))
.body(final_payload)
.send()
.await
.change_context(OpenSearchError::ResponseError)
}
}
}
}
#[async_trait::async_trait]
impl HealthCheck for OpenSearchClient {
async fn deep_health_check(&self) -> CustomResult<(), QueryExecutionError> {
let health = Cluster::new(&self.transport)
.health(ClusterHealthParts::None)
.send()
.await
.change_context(QueryExecutionError::DatabaseError)?
.json::<OpenSearchHealth>()
.await
.change_context(QueryExecutionError::DatabaseError)?;
if health.status != OpenSearchHealthStatus::Red {
Ok(())
} else {
Err(QueryExecutionError::DatabaseError.into())
}
}
}
impl OpenSearchIndexes {
pub fn validate(&self) -> Result<(), ApplicationError> {
use common_utils::{ext_traits::ConfigExt, fp_utils::when};
when(self.payment_attempts.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Payment Attempts index must not be empty".into(),
))
})?;
when(self.payment_intents.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Payment Intents index must not be empty".into(),
))
})?;
when(self.refunds.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Refunds index must not be empty".into(),
))
})?;
when(self.disputes.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Disputes index must not be empty".into(),
))
})?;
Ok(())
}
}
impl OpenSearchAuth {
pub fn validate(&self) -> Result<(), ApplicationError> {
use common_utils::{ext_traits::ConfigExt, fp_utils::when};
match self {
Self::Basic { username, password } => {
when(username.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Basic auth username must not be empty".into(),
))
})?;
when(password.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Basic auth password must not be empty".into(),
))
})?;
}
Self::Aws { region } => {
when(region.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch Aws auth region must not be empty".into(),
))
})?;
}
};
Ok(())
}
}
impl OpenSearchConfig {
pub async fn get_opensearch_client(&self) -> StorageResult<OpenSearchClient> {
Ok(OpenSearchClient::create(self)
.await
.map_err(|_| StorageError::InitializationError)?)
}
pub fn validate(&self) -> Result<(), ApplicationError> {
use common_utils::{ext_traits::ConfigExt, fp_utils::when};
when(self.host.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"Opensearch host must not be empty".into(),
))
})?;
self.indexes.validate()?;
self.auth.validate()?;
Ok(())
}
}
#[derive(Debug, serde::Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum OpenSearchHealthStatus {
Red,
Green,
Yellow,
}
#[derive(Debug, serde::Deserialize)]
pub struct OpenSearchHealth {
pub status: OpenSearchHealthStatus,
}
#[derive(Debug, Clone)]
pub enum OpenSearchQuery {
Msearch,
Search(SearchIndex),
}
#[derive(Debug, Clone)]
pub struct OpenSearchQueryBuilder {
pub query_type: OpenSearchQuery,
pub query: String,
pub offset: Option<i64>,
pub count: Option<i64>,
pub filters: Vec<(String, String)>,
}
impl OpenSearchQueryBuilder {
pub fn new(query_type: OpenSearchQuery, query: String) -> Self {
Self {
query_type,
query,
offset: Default::default(),
count: Default::default(),
filters: Default::default(),
}
}
pub fn set_offset_n_count(&mut self, offset: i64, count: i64) -> QueryResult<()> {
self.offset = Some(offset);
self.count = Some(count);
Ok(())
}
pub fn add_filter_clause(&mut self, lhs: String, rhs: String) -> QueryResult<()> {
self.filters.push((lhs, rhs));
Ok(())
}
pub fn construct_payload(&self, indexes: Vec<SearchIndex>) -> QueryResult<Vec<Value>> {
let mut query =
vec![json!({"multi_match": {"type": "phrase", "query": self.query, "lenient": true}})];
let mut filters = self
.filters
.iter()
.map(|(k, v)| json!({"match_phrase" : {k : v}}))
.collect::<Vec<Value>>();
query.append(&mut filters);
// TODO add index specific filters
Ok(indexes
.iter()
.map(|_index| json!({"query": {"bool": {"filter": query}}}))
.collect::<Vec<Value>>())
}
}

133
crates/analytics/src/search.rs
View File

@ -2,99 +2,33 @@ use api_models::analytics::search::{
GetGlobalSearchRequest, GetSearchRequestWithIndex, GetSearchResponse, OpenMsearchOutput,
OpensearchOutput, SearchIndex,
};
use aws_config::{self, meta::region::RegionProviderChain, Region};
use common_utils::errors::CustomResult;
use opensearch::{
auth::Credentials,
cert::CertificateValidation,
http::{
request::JsonBody,
transport::{SingleNodeConnectionPool, TransportBuilder},
Url,
},
MsearchParts, OpenSearch, SearchParts,
};
use serde_json::{json, Value};
use common_utils::errors::{CustomResult, ReportSwitchExt};
use error_stack::ResultExt;
use serde_json::Value;
use strum::IntoEnumIterator;
use crate::{errors::AnalyticsError, OpensearchAuth, OpensearchConfig, OpensearchIndexes};
#[derive(Debug, thiserror::Error)]
pub enum OpensearchError {
#[error("Opensearch connection error")]
ConnectionError,
#[error("Opensearch NON-200 response content: '{0}'")]
ResponseNotOK(String),
#[error("Opensearch response error")]
ResponseError,
}
pub fn search_index_to_opensearch_index(index: SearchIndex, config: &OpensearchIndexes) -> String {
match index {
SearchIndex::PaymentAttempts => config.payment_attempts.clone(),
SearchIndex::PaymentIntents => config.payment_intents.clone(),
SearchIndex::Refunds => config.refunds.clone(),
SearchIndex::Disputes => config.disputes.clone(),
}
}
async fn get_opensearch_client(config: OpensearchConfig) -> Result<OpenSearch, OpensearchError> {
let url = Url::parse(&config.host).map_err(|_| OpensearchError::ConnectionError)?;
let transport = match config.auth {
OpensearchAuth::Basic { username, password } => {
let credentials = Credentials::Basic(username, password);
TransportBuilder::new(SingleNodeConnectionPool::new(url))
.cert_validation(CertificateValidation::None)
.auth(credentials)
.build()
.map_err(|_| OpensearchError::ConnectionError)?
}
OpensearchAuth::Aws { region } => {
let region_provider = RegionProviderChain::first_try(Region::new(region));
let sdk_config = aws_config::from_env().region(region_provider).load().await;
let conn_pool = SingleNodeConnectionPool::new(url);
TransportBuilder::new(conn_pool)
.auth(
sdk_config
.clone()
.try_into()
.map_err(|_| OpensearchError::ConnectionError)?,
)
.service_name("es")
.build()
.map_err(|_| OpensearchError::ConnectionError)?
}
};
Ok(OpenSearch::new(transport))
}
use crate::opensearch::{
OpenSearchClient, OpenSearchError, OpenSearchQuery, OpenSearchQueryBuilder,
};
pub async fn msearch_results(
client: &OpenSearchClient,
req: GetGlobalSearchRequest,
merchant_id: &String,
config: OpensearchConfig,
) -> CustomResult<Vec<GetSearchResponse>, AnalyticsError> {
let client = get_opensearch_client(config.clone())
) -> CustomResult<Vec<GetSearchResponse>, OpenSearchError> {
let mut query_builder = OpenSearchQueryBuilder::new(OpenSearchQuery::Msearch, req.query);
query_builder
.add_filter_clause("merchant_id".to_string(), merchant_id.to_string())
.switch()?;
let response_body = client
.execute(query_builder)
.await
.map_err(|_| AnalyticsError::UnknownError)?;
let mut msearch_vector: Vec<JsonBody<Value>> = vec![];
for index in SearchIndex::iter() {
msearch_vector
.push(json!({"index": search_index_to_opensearch_index(index,&config.indexes)}).into());
msearch_vector.push(json!({"query": {"bool": {"filter": [{"multi_match": {"type": "phrase", "query": req.query, "lenient": true}},{"match_phrase": {"merchant_id": merchant_id}}]}}}).into());
}
let response = client
.msearch(MsearchParts::None)
.body(msearch_vector)
.send()
.await
.map_err(|_| AnalyticsError::UnknownError)?;
let response_body = response
.change_context(OpenSearchError::ConnectionError)?
.json::<OpenMsearchOutput<Value>>()
.await
.map_err(|_| AnalyticsError::UnknownError)?;
.change_context(OpenSearchError::ResponseError)?;
Ok(response_body
.responses
@ -114,29 +48,30 @@ pub async fn msearch_results(
}
pub async fn search_results(
client: &OpenSearchClient,
req: GetSearchRequestWithIndex,
merchant_id: &String,
config: OpensearchConfig,
) -> CustomResult<GetSearchResponse, AnalyticsError> {
) -> CustomResult<GetSearchResponse, OpenSearchError> {
let search_req = req.search_req;
let client = get_opensearch_client(config.clone())
.await
.map_err(|_| AnalyticsError::UnknownError)?;
let mut query_builder =
OpenSearchQueryBuilder::new(OpenSearchQuery::Search(req.index), search_req.query);
let response = client
.search(SearchParts::Index(&[&search_index_to_opensearch_index(req.index.clone(),&config.indexes)]))
.from(search_req.offset)
.size(search_req.count)
.body(json!({"query": {"bool": {"filter": [{"multi_match": {"type": "phrase", "query": search_req.query, "lenient": true}},{"match_phrase": {"merchant_id": merchant_id}}]}}}))
.send()
.await
.map_err(|_| AnalyticsError::UnknownError)?;
query_builder
.add_filter_clause("merchant_id".to_string(), merchant_id.to_string())
.switch()?;
let response_body = response
query_builder
.set_offset_n_count(search_req.offset, search_req.count)
.switch()?;
let response_body = client
.execute(query_builder)
.await
.change_context(OpenSearchError::ConnectionError)?
.json::<OpensearchOutput<Value>>()
.await
.map_err(|_| AnalyticsError::UnknownError)?;
.change_context(OpenSearchError::ResponseError)?;
Ok(GetSearchResponse {
count: response_body.hits.total.value,