feat(analytics): Add Clickhouse based analytics (#2988)

Co-authored-by: harsh_sharma_juspay <harsh.sharma@juspay.in>
Co-authored-by: Ivor Dsouza <ivor.dsouza@juspay.in>
Co-authored-by: Chethan Rao <70657455+Chethan-rao@users.noreply.github.com>
Co-authored-by: nain-F49FF806 <126972030+nain-F49FF806@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com>
Co-authored-by: akshay.s <akshay.s@juspay.in>
Co-authored-by: Gnanasundari24 <118818938+Gnanasundari24@users.noreply.github.com>

crates/api_models/src/analytics/api_event.rs

@@ -0,0 +1,148 @@
+use std::{
+    collections::hash_map::DefaultHasher,
+    hash::{Hash, Hasher},
+};
+
+use super::{NameDescription, TimeRange};
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+pub struct ApiLogsRequest {
+    #[serde(flatten)]
+    pub query_param: QueryType,
+    pub api_name_filter: Option<Vec<String>>,
+}
+
+pub enum FilterType {
+    ApiCountFilter,
+    LatencyFilter,
+    StatusCodeFilter,
+}
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(tag = "type")]
+pub enum QueryType {
+    Payment {
+        payment_id: String,
+    },
+    Refund {
+        payment_id: String,
+        refund_id: String,
+    },
+}
+
+#[derive(
+    Debug,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::AsRefStr,
+    PartialEq,
+    PartialOrd,
+    Eq,
+    Ord,
+    strum::Display,
+    strum::EnumIter,
+    Clone,
+    Copy,
+)]
+#[serde(rename_all = "snake_case")]
+#[strum(serialize_all = "snake_case")]
+pub enum ApiEventDimensions {
+    // Do not change the order of these enums
+    // Consult the Dashboard FE folks since these also affects the order of metrics on FE
+    StatusCode,
+    FlowType,
+    ApiFlow,
+}
+
+impl From<ApiEventDimensions> for NameDescription {
+    fn from(value: ApiEventDimensions) -> Self {
+        Self {
+            name: value.to_string(),
+            desc: String::new(),
+        }
+    }
+}
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
+pub struct ApiEventFilters {
+    pub status_code: Vec<u64>,
+    pub flow_type: Vec<String>,
+    pub api_flow: Vec<String>,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::Display,
+    strum::EnumIter,
+    strum::AsRefStr,
+)]
+#[strum(serialize_all = "snake_case")]
+#[serde(rename_all = "snake_case")]
+pub enum ApiEventMetrics {
+    Latency,
+    ApiCount,
+    StatusCodeCount,
+}
+
+impl From<ApiEventMetrics> for NameDescription {
+    fn from(value: ApiEventMetrics) -> Self {
+        Self {
+            name: value.to_string(),
+            desc: String::new(),
+        }
+    }
+}
+
+#[derive(Debug, serde::Serialize, Eq)]
+pub struct ApiEventMetricsBucketIdentifier {
+    #[serde(rename = "time_range")]
+    pub time_bucket: TimeRange,
+    // Coz FE sucks
+    #[serde(rename = "time_bucket")]
+    #[serde(with = "common_utils::custom_serde::iso8601custom")]
+    pub start_time: time::PrimitiveDateTime,
+}
+
+impl ApiEventMetricsBucketIdentifier {
+    pub fn new(normalized_time_range: TimeRange) -> Self {
+        Self {
+            time_bucket: normalized_time_range,
+            start_time: normalized_time_range.start_time,
+        }
+    }
+}
+
+impl Hash for ApiEventMetricsBucketIdentifier {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.time_bucket.hash(state);
+    }
+}
+
+impl PartialEq for ApiEventMetricsBucketIdentifier {
+    fn eq(&self, other: &Self) -> bool {
+        let mut left = DefaultHasher::new();
+        self.hash(&mut left);
+        let mut right = DefaultHasher::new();
+        other.hash(&mut right);
+        left.finish() == right.finish()
+    }
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct ApiEventMetricsBucketValue {
+    pub latency: Option<u64>,
+    pub api_count: Option<u64>,
+    pub status_code_count: Option<u64>,
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct ApiMetricsBucketResponse {
+    #[serde(flatten)]
+    pub values: ApiEventMetricsBucketValue,
+    #[serde(flatten)]
+    pub dimensions: ApiEventMetricsBucketIdentifier,
+}

crates/api_models/src/analytics/payments.rs

@@ -3,13 +3,12 @@ use std::{
     hash::{Hash, Hasher},
 };
 
-use common_enums::enums::{AttemptStatus, AuthenticationType, Currency, PaymentMethod};
-use common_utils::events::ApiEventMetric;
-
 use super::{NameDescription, TimeRange};
-use crate::{analytics::MetricsResponse, enums::Connector};
+use crate::enums::{
+    AttemptStatus, AuthenticationType, Connector, Currency, PaymentMethod, PaymentMethodType,
+};
 
-#[derive(Clone, Debug, Default, serde::Deserialize, masking::Serialize)]
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
 pub struct PaymentFilters {
     #[serde(default)]
     pub currency: Vec<Currency>,
@@ -21,6 +20,8 @@ pub struct PaymentFilters {
     pub auth_type: Vec<AuthenticationType>,
     #[serde(default)]
     pub payment_method: Vec<PaymentMethod>,
+    #[serde(default)]
+    pub payment_method_type: Vec<PaymentMethodType>,
 }
 
 #[derive(
@@ -44,6 +45,7 @@ pub enum PaymentDimensions {
     // Consult the Dashboard FE folks since these also affects the order of metrics on FE
     Connector,
     PaymentMethod,
+    PaymentMethodType,
     Currency,
     #[strum(serialize = "authentication_type")]
     #[serde(rename = "authentication_type")]
@@ -73,6 +75,35 @@ pub enum PaymentMetrics {
     PaymentSuccessCount,
     PaymentProcessedAmount,
     AvgTicketSize,
+    RetriesCount,
+    ConnectorSuccessRate,
+}
+
+#[derive(Debug, Default, serde::Serialize)]
+pub struct ErrorResult {
+    pub reason: String,
+    pub count: i64,
+    pub percentage: f64,
+}
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::Display,
+    strum::EnumIter,
+    strum::AsRefStr,
+)]
+#[strum(serialize_all = "snake_case")]
+#[serde(rename_all = "snake_case")]
+pub enum PaymentDistributions {
+    #[strum(serialize = "error_message")]
+    PaymentErrorMessage,
 }
 
 pub mod metric_behaviour {
@@ -109,6 +140,7 @@ pub struct PaymentMetricsBucketIdentifier {
     #[serde(rename = "authentication_type")]
     pub auth_type: Option<AuthenticationType>,
     pub payment_method: Option<String>,
+    pub payment_method_type: Option<String>,
     #[serde(rename = "time_range")]
     pub time_bucket: TimeRange,
     // Coz FE sucks
@@ -124,6 +156,7 @@ impl PaymentMetricsBucketIdentifier {
         connector: Option<String>,
         auth_type: Option<AuthenticationType>,
         payment_method: Option<String>,
+        payment_method_type: Option<String>,
         normalized_time_range: TimeRange,
     ) -> Self {
         Self {
@@ -132,6 +165,7 @@ impl PaymentMetricsBucketIdentifier {
             connector,
             auth_type,
             payment_method,
+            payment_method_type,
             time_bucket: normalized_time_range,
             start_time: normalized_time_range.start_time,
         }
@@ -145,6 +179,7 @@ impl Hash for PaymentMetricsBucketIdentifier {
         self.connector.hash(state);
         self.auth_type.map(|i| i.to_string()).hash(state);
         self.payment_method.hash(state);
+        self.payment_method_type.hash(state);
         self.time_bucket.hash(state);
     }
 }
@@ -166,6 +201,10 @@ pub struct PaymentMetricsBucketValue {
     pub payment_success_count: Option<u64>,
     pub payment_processed_amount: Option<u64>,
     pub avg_ticket_size: Option<f64>,
+    pub payment_error_message: Option<Vec<ErrorResult>>,
+    pub retries_count: Option<u64>,
+    pub retries_amount_processed: Option<u64>,
+    pub connector_success_rate: Option<f64>,
 }
 
 #[derive(Debug, serde::Serialize)]
@@ -175,6 +214,3 @@ pub struct MetricsBucketResponse {
     #[serde(flatten)]
     pub dimensions: PaymentMetricsBucketIdentifier,
 }
-
-impl ApiEventMetric for MetricsBucketResponse {}
-impl ApiEventMetric for MetricsResponse<MetricsBucketResponse> {}

crates/api_models/src/analytics/refunds.rs

@@ -3,10 +3,7 @@ use std::{
     hash::{Hash, Hasher},
 };
 
-use common_enums::enums::{Currency, RefundStatus};
-use common_utils::events::ApiEventMetric;
-
-use crate::analytics::MetricsResponse;
+use crate::{enums::Currency, refunds::RefundStatus};
 
 #[derive(
     Clone,
@@ -20,7 +17,7 @@ use crate::analytics::MetricsResponse;
     strum::Display,
     strum::EnumString,
 )]
-// TODO RefundType common_enums need to mapped to storage_model
+// TODO RefundType api_models_oss need to mapped to storage_model
 #[serde(rename_all = "snake_case")]
 #[strum(serialize_all = "snake_case")]
 pub enum RefundType {
@@ -31,7 +28,7 @@ pub enum RefundType {
 }
 
 use super::{NameDescription, TimeRange};
-#[derive(Clone, Debug, Default, serde::Deserialize, masking::Serialize)]
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
 pub struct RefundFilters {
     #[serde(default)]
     pub currency: Vec<Currency>,
@@ -115,8 +112,9 @@ impl From<RefundDimensions> for NameDescription {
 #[derive(Debug, serde::Serialize, Eq)]
 pub struct RefundMetricsBucketIdentifier {
     pub currency: Option<Currency>,
-    pub refund_status: Option<RefundStatus>,
+    pub refund_status: Option<String>,
     pub connector: Option<String>,
+
     pub refund_type: Option<String>,
     #[serde(rename = "time_range")]
     pub time_bucket: TimeRange,
@@ -128,7 +126,7 @@ pub struct RefundMetricsBucketIdentifier {
 impl Hash for RefundMetricsBucketIdentifier {
     fn hash<H: Hasher>(&self, state: &mut H) {
         self.currency.hash(state);
-        self.refund_status.map(|i| i.to_string()).hash(state);
+        self.refund_status.hash(state);
         self.connector.hash(state);
         self.refund_type.hash(state);
         self.time_bucket.hash(state);
@@ -147,7 +145,7 @@ impl PartialEq for RefundMetricsBucketIdentifier {
 impl RefundMetricsBucketIdentifier {
     pub fn new(
         currency: Option<Currency>,
-        refund_status: Option<RefundStatus>,
+        refund_status: Option<String>,
         connector: Option<String>,
         refund_type: Option<String>,
         normalized_time_range: TimeRange,
@@ -162,7 +160,6 @@ impl RefundMetricsBucketIdentifier {
         }
     }
 }
-
 #[derive(Debug, serde::Serialize)]
 pub struct RefundMetricsBucketValue {
     pub refund_success_rate: Option<f64>,
@@ -170,7 +167,6 @@ pub struct RefundMetricsBucketValue {
     pub refund_success_count: Option<u64>,
     pub refund_processed_amount: Option<u64>,
 }
-
 #[derive(Debug, serde::Serialize)]
 pub struct RefundMetricsBucketResponse {
     #[serde(flatten)]
@@ -178,6 +174,3 @@ pub struct RefundMetricsBucketResponse {
     #[serde(flatten)]
     pub dimensions: RefundMetricsBucketIdentifier,
 }
-
-impl ApiEventMetric for RefundMetricsBucketResponse {}
-impl ApiEventMetric for MetricsResponse<RefundMetricsBucketResponse> {}

crates/api_models/src/analytics/sdk_events.rs

@@ -0,0 +1,215 @@
+use std::{
+    collections::hash_map::DefaultHasher,
+    hash::{Hash, Hasher},
+};
+
+use super::{NameDescription, TimeRange};
+
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SdkEventsRequest {
+    pub payment_id: String,
+    pub time_range: TimeRange,
+}
+
+#[derive(Clone, Debug, Default, serde::Deserialize, serde::Serialize)]
+pub struct SdkEventFilters {
+    #[serde(default)]
+    pub payment_method: Vec<String>,
+    #[serde(default)]
+    pub platform: Vec<String>,
+    #[serde(default)]
+    pub browser_name: Vec<String>,
+    #[serde(default)]
+    pub source: Vec<String>,
+    #[serde(default)]
+    pub component: Vec<String>,
+    #[serde(default)]
+    pub payment_experience: Vec<String>,
+}
+
+#[derive(
+    Debug,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::AsRefStr,
+    PartialEq,
+    PartialOrd,
+    Eq,
+    Ord,
+    strum::Display,
+    strum::EnumIter,
+    Clone,
+    Copy,
+)]
+#[serde(rename_all = "snake_case")]
+#[strum(serialize_all = "snake_case")]
+pub enum SdkEventDimensions {
+    // Do not change the order of these enums
+    // Consult the Dashboard FE folks since these also affects the order of metrics on FE
+    PaymentMethod,
+    Platform,
+    BrowserName,
+    Source,
+    Component,
+    PaymentExperience,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::Display,
+    strum::EnumIter,
+    strum::AsRefStr,
+)]
+#[strum(serialize_all = "snake_case")]
+#[serde(rename_all = "snake_case")]
+pub enum SdkEventMetrics {
+    PaymentAttempts,
+    PaymentSuccessCount,
+    PaymentMethodsCallCount,
+    SdkRenderedCount,
+    SdkInitiatedCount,
+    PaymentMethodSelectedCount,
+    PaymentDataFilledCount,
+    AveragePaymentTime,
+}
+
+#[derive(
+    Clone,
+    Debug,
+    Hash,
+    PartialEq,
+    Eq,
+    serde::Serialize,
+    serde::Deserialize,
+    strum::Display,
+    strum::EnumIter,
+    strum::AsRefStr,
+)]
+#[strum(serialize_all = "SCREAMING_SNAKE_CASE")]
+#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+pub enum SdkEventNames {
+    StripeElementsCalled,
+    AppRendered,
+    PaymentMethodChanged,
+    PaymentDataFilled,
+    PaymentAttempt,
+    PaymentSuccess,
+    PaymentMethodsCall,
+    ConfirmCall,
+    SessionsCall,
+    CustomerPaymentMethodsCall,
+    RedirectingUser,
+    DisplayBankTransferInfoPage,
+    DisplayQrCodeInfoPage,
+}
+
+pub mod metric_behaviour {
+    pub struct PaymentAttempts;
+    pub struct PaymentSuccessCount;
+    pub struct PaymentMethodsCallCount;
+    pub struct SdkRenderedCount;
+    pub struct SdkInitiatedCount;
+    pub struct PaymentMethodSelectedCount;
+    pub struct PaymentDataFilledCount;
+    pub struct AveragePaymentTime;
+}
+
+impl From<SdkEventMetrics> for NameDescription {
+    fn from(value: SdkEventMetrics) -> Self {
+        Self {
+            name: value.to_string(),
+            desc: String::new(),
+        }
+    }
+}
+
+impl From<SdkEventDimensions> for NameDescription {
+    fn from(value: SdkEventDimensions) -> Self {
+        Self {
+            name: value.to_string(),
+            desc: String::new(),
+        }
+    }
+}
+
+#[derive(Debug, serde::Serialize, Eq)]
+pub struct SdkEventMetricsBucketIdentifier {
+    pub payment_method: Option<String>,
+    pub platform: Option<String>,
+    pub browser_name: Option<String>,
+    pub source: Option<String>,
+    pub component: Option<String>,
+    pub payment_experience: Option<String>,
+    pub time_bucket: Option<String>,
+}
+
+impl SdkEventMetricsBucketIdentifier {
+    pub fn new(
+        payment_method: Option<String>,
+        platform: Option<String>,
+        browser_name: Option<String>,
+        source: Option<String>,
+        component: Option<String>,
+        payment_experience: Option<String>,
+        time_bucket: Option<String>,
+    ) -> Self {
+        Self {
+            payment_method,
+            platform,
+            browser_name,
+            source,
+            component,
+            payment_experience,
+            time_bucket,
+        }
+    }
+}
+
+impl Hash for SdkEventMetricsBucketIdentifier {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.payment_method.hash(state);
+        self.platform.hash(state);
+        self.browser_name.hash(state);
+        self.source.hash(state);
+        self.component.hash(state);
+        self.payment_experience.hash(state);
+        self.time_bucket.hash(state);
+    }
+}
+
+impl PartialEq for SdkEventMetricsBucketIdentifier {
+    fn eq(&self, other: &Self) -> bool {
+        let mut left = DefaultHasher::new();
+        self.hash(&mut left);
+        let mut right = DefaultHasher::new();
+        other.hash(&mut right);
+        left.finish() == right.finish()
+    }
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct SdkEventMetricsBucketValue {
+    pub payment_attempts: Option<u64>,
+    pub payment_success_count: Option<u64>,
+    pub payment_methods_call_count: Option<u64>,
+    pub average_payment_time: Option<f64>,
+    pub sdk_rendered_count: Option<u64>,
+    pub sdk_initiated_count: Option<u64>,
+    pub payment_method_selected_count: Option<u64>,
+    pub payment_data_filled_count: Option<u64>,
+}
+
+#[derive(Debug, serde::Serialize)]
+pub struct MetricsBucketResponse {
+    #[serde(flatten)]
+    pub values: SdkEventMetricsBucketValue,
+    #[serde(flatten)]
+    pub dimensions: SdkEventMetricsBucketIdentifier,
+}