refactor(scheduler): move scheduler to new crate to support workflows in multiple crates (#1681)

This commit is contained in:
Jagan
2023-09-06 13:11:09 +05:30
committed by GitHub
parent ec4652ec09
commit d4221f3368
72 changed files with 1966 additions and 1124 deletions

View File

@ -0,0 +1,59 @@
use bb8::PooledConnection;
use common_utils::errors;
use diesel::PgConnection;
use error_stack::{IntoReport, ResultExt};
pub type PgPool = bb8::Pool<async_bb8_diesel::ConnectionManager<PgConnection>>;
pub type PgPooledConn = async_bb8_diesel::Connection<PgConnection>;
#[allow(clippy::expect_used)]
pub async fn redis_connection(
redis: &redis_interface::RedisSettings,
) -> redis_interface::RedisConnectionPool {
redis_interface::RedisConnectionPool::new(redis)
.await
.expect("Failed to create Redis Connection Pool")
}
pub async fn pg_connection_read<T: crate::DatabaseStore>(
store: &T,
) -> errors::CustomResult<
PooledConnection<'_, async_bb8_diesel::ConnectionManager<PgConnection>>,
crate::errors::StorageError,
> {
// If only OLAP is enabled get replica pool.
#[cfg(all(feature = "olap", not(feature = "oltp")))]
let pool = store.get_replica_pool();
// If either one of these are true we need to get master pool.
// 1. Only OLTP is enabled.
// 2. Both OLAP and OLTP is enabled.
// 3. Both OLAP and OLTP is disabled.
#[cfg(any(
all(not(feature = "olap"), feature = "oltp"),
all(feature = "olap", feature = "oltp"),
all(not(feature = "olap"), not(feature = "oltp"))
))]
let pool = store.get_master_pool();
pool.get()
.await
.into_report()
.change_context(crate::errors::StorageError::DatabaseConnectionError)
}
pub async fn pg_connection_write<T: crate::DatabaseStore>(
store: &T,
) -> errors::CustomResult<
PooledConnection<'_, async_bb8_diesel::ConnectionManager<PgConnection>>,
crate::errors::StorageError,
> {
// Since all writes should happen to master DB only choose master DB.
let pool = store.get_master_pool();
pool.get()
.await
.into_report()
.change_context(crate::errors::StorageError::DatabaseConnectionError)
}

View File

@ -0,0 +1,366 @@
use std::fmt::Display;
use actix_web::ResponseError;
use common_utils::errors::ErrorSwitch;
use config::ConfigError;
use data_models::errors::StorageError as DataStorageError;
use http::StatusCode;
pub use redis_interface::errors::RedisError;
use router_env::opentelemetry::metrics::MetricsError;
use crate::{errors as storage_errors, store::errors::DatabaseError};
pub type ApplicationResult<T> = Result<T, ApplicationError>;
macro_rules! impl_error_display {
($st: ident, $arg: tt) => {
impl Display for $st {
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
fmt,
"{{ error_type: {:?}, error_description: {} }}",
self, $arg
)
}
}
};
}
macro_rules! impl_error_type {
($name: ident, $arg: tt) => {
#[derive(Debug)]
pub struct $name;
impl_error_display!($name, $arg);
impl std::error::Error for $name {}
};
}
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
#[error("DatabaseError: {0:?}")]
DatabaseError(error_stack::Report<DatabaseError>),
#[error("ValueNotFound: {0}")]
ValueNotFound(String),
#[error("DuplicateValue: {entity} already exists {key:?}")]
DuplicateValue {
entity: &'static str,
key: Option<String>,
},
#[error("Timed out while trying to connect to the database")]
DatabaseConnectionError,
#[error("KV error")]
KVError,
#[error("Serialization failure")]
SerializationFailed,
#[error("MockDb error")]
MockDbError,
#[error("Customer with this id is Redacted")]
CustomerRedacted,
#[error("Deserialization failure")]
DeserializationFailed,
#[error("Error while encrypting data")]
EncryptionError,
#[error("Error while decrypting data from database")]
DecryptionError,
#[error("RedisError: {0:?}")]
RedisError(error_stack::Report<RedisError>),
}
impl ErrorSwitch<DataStorageError> for StorageError {
fn switch(&self) -> DataStorageError {
self.into()
}
}
#[allow(clippy::from_over_into)]
impl Into<DataStorageError> for &StorageError {
fn into(self) -> DataStorageError {
match self {
StorageError::DatabaseError(i) => match i.current_context() {
storage_errors::DatabaseError::DatabaseConnectionError => {
DataStorageError::DatabaseConnectionError
}
// TODO: Update this error type to encompass & propagate the missing type (instead of generic `db value not found`)
storage_errors::DatabaseError::NotFound => {
DataStorageError::ValueNotFound(String::from("db value not found"))
}
// TODO: Update this error type to encompass & propagate the duplicate type (instead of generic `db value not found`)
storage_errors::DatabaseError::UniqueViolation => {
DataStorageError::DuplicateValue {
entity: "db entity",
key: None,
}
}
storage_errors::DatabaseError::NoFieldsToUpdate => {
DataStorageError::DatabaseError("No fields to update".to_string())
}
storage_errors::DatabaseError::QueryGenerationFailed => {
DataStorageError::DatabaseError("Query generation failed".to_string())
}
storage_errors::DatabaseError::Others => {
DataStorageError::DatabaseError("Unknown database error".to_string())
}
},
StorageError::ValueNotFound(i) => DataStorageError::ValueNotFound(i.clone()),
StorageError::DuplicateValue { entity, key } => DataStorageError::DuplicateValue {
entity,
key: key.clone(),
},
StorageError::DatabaseConnectionError => DataStorageError::DatabaseConnectionError,
StorageError::KVError => DataStorageError::KVError,
StorageError::SerializationFailed => DataStorageError::SerializationFailed,
StorageError::MockDbError => DataStorageError::MockDbError,
StorageError::CustomerRedacted => DataStorageError::CustomerRedacted,
StorageError::DeserializationFailed => DataStorageError::DeserializationFailed,
StorageError::EncryptionError => DataStorageError::EncryptionError,
StorageError::DecryptionError => DataStorageError::DecryptionError,
StorageError::RedisError(i) => match i.current_context() {
// TODO: Update this error type to encompass & propagate the missing type (instead of generic `redis value not found`)
RedisError::NotFound => {
DataStorageError::ValueNotFound("redis value not found".to_string())
}
RedisError::JsonSerializationFailed => DataStorageError::SerializationFailed,
RedisError::JsonDeserializationFailed => DataStorageError::DeserializationFailed,
i => DataStorageError::RedisError(format!("{:?}", i)),
},
}
}
}
impl From<error_stack::Report<RedisError>> for StorageError {
fn from(err: error_stack::Report<RedisError>) -> Self {
Self::RedisError(err)
}
}
impl From<error_stack::Report<DatabaseError>> for StorageError {
fn from(err: error_stack::Report<DatabaseError>) -> Self {
Self::DatabaseError(err)
}
}
impl StorageError {
pub fn is_db_not_found(&self) -> bool {
match self {
Self::DatabaseError(err) => matches!(err.current_context(), DatabaseError::NotFound),
_ => false,
}
}
pub fn is_db_unique_violation(&self) -> bool {
match self {
Self::DatabaseError(err) => {
matches!(err.current_context(), DatabaseError::UniqueViolation,)
}
_ => false,
}
}
}
impl_error_type!(EncryptionError, "Encryption error");
#[derive(Debug, thiserror::Error)]
pub enum ApplicationError {
// Display's impl can be overridden by the attribute error marco.
// Don't use Debug here, Debug gives error stack in response.
#[error("Application configuration error: {0}")]
ConfigurationError(ConfigError),
#[error("Invalid configuration value provided: {0}")]
InvalidConfigurationValueError(String),
#[error("Metrics error: {0}")]
MetricsError(MetricsError),
#[error("I/O: {0}")]
IoError(std::io::Error),
#[error("Error while constructing api client: {0}")]
ApiClientError(ApiClientError),
}
impl From<MetricsError> for ApplicationError {
fn from(err: MetricsError) -> Self {
Self::MetricsError(err)
}
}
impl From<std::io::Error> for ApplicationError {
fn from(err: std::io::Error) -> Self {
Self::IoError(err)
}
}
impl From<ring::error::Unspecified> for EncryptionError {
fn from(_: ring::error::Unspecified) -> Self {
Self
}
}
impl From<ConfigError> for ApplicationError {
fn from(err: ConfigError) -> Self {
Self::ConfigurationError(err)
}
}
fn error_response<T: Display>(err: &T) -> actix_web::HttpResponse {
actix_web::HttpResponse::BadRequest()
.content_type(mime::APPLICATION_JSON)
.body(format!(r#"{{ "error": {{ "message": "{err}" }} }}"#))
}
impl ResponseError for ApplicationError {
fn status_code(&self) -> StatusCode {
match self {
Self::MetricsError(_)
| Self::IoError(_)
| Self::ConfigurationError(_)
| Self::InvalidConfigurationValueError(_)
| Self::ApiClientError(_) => StatusCode::INTERNAL_SERVER_ERROR,
}
}
fn error_response(&self) -> actix_web::HttpResponse {
error_response(self)
}
}
#[derive(Debug, thiserror::Error, PartialEq, Clone)]
pub enum ApiClientError {
#[error("Header map construction failed")]
HeaderMapConstructionFailed,
#[error("Invalid proxy configuration")]
InvalidProxyConfiguration,
#[error("Client construction failed")]
ClientConstructionFailed,
#[error("Certificate decode failed")]
CertificateDecodeFailed,
#[error("Request body serialization failed")]
BodySerializationFailed,
#[error("Unexpected state reached/Invariants conflicted")]
UnexpectedState,
#[error("URL encoding of request payload failed")]
UrlEncodingFailed,
#[error("Failed to send request to connector {0}")]
RequestNotSent(String),
#[error("Failed to decode response")]
ResponseDecodingFailed,
#[error("Server responded with Request Timeout")]
RequestTimeoutReceived,
#[error("connection closed before a message could complete")]
ConnectionClosed,
#[error("Server responded with Internal Server Error")]
InternalServerErrorReceived,
#[error("Server responded with Bad Gateway")]
BadGatewayReceived,
#[error("Server responded with Service Unavailable")]
ServiceUnavailableReceived,
#[error("Server responded with Gateway Timeout")]
GatewayTimeoutReceived,
#[error("Server responded with unexpected response")]
UnexpectedServerResponse,
}
impl ApiClientError {
pub fn is_upstream_timeout(&self) -> bool {
self == &Self::RequestTimeoutReceived
}
pub fn is_connection_closed(&self) -> bool {
self == &Self::ConnectionClosed
}
}
#[derive(Debug, thiserror::Error, PartialEq)]
pub enum ConnectorError {
#[error("Error while obtaining URL for the integration")]
FailedToObtainIntegrationUrl,
#[error("Failed to encode connector request")]
RequestEncodingFailed,
#[error("Request encoding failed : {0}")]
RequestEncodingFailedWithReason(String),
#[error("Parsing failed")]
ParsingFailed,
#[error("Failed to deserialize connector response")]
ResponseDeserializationFailed,
#[error("Failed to execute a processing step: {0:?}")]
ProcessingStepFailed(Option<bytes::Bytes>),
#[error("The connector returned an unexpected response: {0:?}")]
UnexpectedResponseError(bytes::Bytes),
#[error("Failed to parse custom routing rules from merchant account")]
RoutingRulesParsingError,
#[error("Failed to obtain preferred connector from merchant account")]
FailedToObtainPreferredConnector,
#[error("An invalid connector name was provided")]
InvalidConnectorName,
#[error("An invalid Wallet was used")]
InvalidWallet,
#[error("Failed to handle connector response")]
ResponseHandlingFailed,
#[error("Missing required field: {field_name}")]
MissingRequiredField { field_name: &'static str },
#[error("Missing required fields: {field_names:?}")]
MissingRequiredFields { field_names: Vec<&'static str> },
#[error("Failed to obtain authentication type")]
FailedToObtainAuthType,
#[error("Failed to obtain certificate")]
FailedToObtainCertificate,
#[error("Connector meta data not found")]
NoConnectorMetaData,
#[error("Failed to obtain certificate key")]
FailedToObtainCertificateKey,
#[error("This step has not been implemented for: {0}")]
NotImplemented(String),
#[error("{message} is not supported by {connector}")]
NotSupported {
message: String,
connector: &'static str,
payment_experience: String,
},
#[error("{flow} flow not supported by {connector} connector")]
FlowNotSupported { flow: String, connector: String },
#[error("Capture method not supported")]
CaptureMethodNotSupported,
#[error("Missing connector transaction ID")]
MissingConnectorTransactionID,
#[error("Missing connector refund ID")]
MissingConnectorRefundID,
#[error("Webhooks not implemented for this connector")]
WebhooksNotImplemented,
#[error("Failed to decode webhook event body")]
WebhookBodyDecodingFailed,
#[error("Signature not found for incoming webhook")]
WebhookSignatureNotFound,
#[error("Failed to verify webhook source")]
WebhookSourceVerificationFailed,
#[error("Could not find merchant secret in DB for incoming webhook source verification")]
WebhookVerificationSecretNotFound,
#[error("Incoming webhook object reference ID not found")]
WebhookReferenceIdNotFound,
#[error("Incoming webhook event type not found")]
WebhookEventTypeNotFound,
#[error("Incoming webhook event resource object not found")]
WebhookResourceObjectNotFound,
#[error("Could not respond to the incoming webhook event")]
WebhookResponseEncodingFailed,
#[error("Invalid Date/time format")]
InvalidDateFormat,
#[error("Date Formatting Failed")]
DateFormattingFailed,
#[error("Invalid Data format")]
InvalidDataFormat { field_name: &'static str },
#[error("Payment Method data / Payment Method Type / Payment Experience Mismatch ")]
MismatchedPaymentData,
#[error("Failed to parse Wallet token")]
InvalidWalletToken,
#[error("Missing Connector Related Transaction ID")]
MissingConnectorRelatedTransactionID { id: String },
#[error("File Validation failed")]
FileValidationFailed { reason: String },
#[error("Missing 3DS redirection payload: {field_name}")]
MissingConnectorRedirectionPayload { field_name: &'static str },
}

View File

@ -1,11 +1,19 @@
use std::sync::Arc;
use data_models::errors::{StorageError, StorageResult};
use common_utils::errors::CustomResult;
use data_models::{
errors::{StorageError, StorageResult},
payments::payment_intent::PaymentIntent,
};
use diesel_models::{self as store};
use error_stack::ResultExt;
use futures::lock::Mutex;
use masking::StrongSecret;
use redis::{kv_store::RedisConnInterface, RedisStore};
pub mod config;
pub mod connection;
pub mod database;
pub mod errors;
pub mod metrics;
pub mod payments;
pub mod redis;
@ -206,6 +214,58 @@ impl<T: DatabaseStore> KVRouterStore<T> {
}
}
#[derive(Clone)]
pub struct MockDb {
pub addresses: Arc<Mutex<Vec<store::Address>>>,
pub configs: Arc<Mutex<Vec<store::Config>>>,
pub merchant_accounts: Arc<Mutex<Vec<store::MerchantAccount>>>,
pub merchant_connector_accounts: Arc<Mutex<Vec<store::MerchantConnectorAccount>>>,
pub payment_attempts: Arc<Mutex<Vec<store::PaymentAttempt>>>,
pub payment_intents: Arc<Mutex<Vec<PaymentIntent>>>,
pub payment_methods: Arc<Mutex<Vec<store::PaymentMethod>>>,
pub customers: Arc<Mutex<Vec<store::Customer>>>,
pub refunds: Arc<Mutex<Vec<store::Refund>>>,
pub processes: Arc<Mutex<Vec<store::ProcessTracker>>>,
pub connector_response: Arc<Mutex<Vec<store::ConnectorResponse>>>,
// pub redis: Arc<redis_interface::RedisConnectionPool>,
pub api_keys: Arc<Mutex<Vec<store::ApiKey>>>,
pub ephemeral_keys: Arc<Mutex<Vec<store::EphemeralKey>>>,
pub cards_info: Arc<Mutex<Vec<store::CardInfo>>>,
pub events: Arc<Mutex<Vec<store::Event>>>,
pub disputes: Arc<Mutex<Vec<store::Dispute>>>,
pub lockers: Arc<Mutex<Vec<store::LockerMockUp>>>,
pub mandates: Arc<Mutex<Vec<store::Mandate>>>,
pub captures: Arc<Mutex<Vec<crate::store::capture::Capture>>>,
pub merchant_key_store: Arc<Mutex<Vec<crate::store::merchant_key_store::MerchantKeyStore>>>,
}
impl MockDb {
pub async fn new() -> Self {
Self {
addresses: Default::default(),
configs: Default::default(),
merchant_accounts: Default::default(),
merchant_connector_accounts: Default::default(),
payment_attempts: Default::default(),
payment_intents: Default::default(),
payment_methods: Default::default(),
customers: Default::default(),
refunds: Default::default(),
processes: Default::default(),
connector_response: Default::default(),
// redis: Arc::new(crate::connection::redis_connection(&redis).await),
api_keys: Default::default(),
ephemeral_keys: Default::default(),
cards_info: Default::default(),
events: Default::default(),
disputes: Default::default(),
lockers: Default::default(),
mandates: Default::default(),
captures: Default::default(),
merchant_key_store: Default::default(),
}
}
}
// TODO: This should not be used beyond this crate
// Remove the pub modified once StorageScheme usage is completed
pub trait DataModelExt {
@ -232,6 +292,14 @@ impl DataModelExt for data_models::MerchantStorageScheme {
}
}
impl RedisConnInterface for MockDb {
fn get_redis_conn(
&self,
) -> Result<Arc<redis_interface::RedisConnectionPool>, error_stack::Report<RedisError>> {
Err(RedisError::RedisConnectionError.into())
}
}
pub(crate) fn diesel_error_to_data_error(
diesel_error: &diesel_models::errors::DatabaseError,
) -> StorageError {

View File

@ -36,7 +36,7 @@ use router_env::logger;
use crate::{
redis::kv_store::{PartitionKey, RedisConnInterface},
utils::{pg_connection_read, pg_connection_write},
DataModelExt, DatabaseStore, KVRouterStore,
CustomResult, DataModelExt, DatabaseStore, KVRouterStore, MockDb,
};
#[async_trait::async_trait]
@ -662,6 +662,137 @@ impl<T: DatabaseStore> PaymentIntentInterface for crate::RouterStore<T> {
}
}
#[async_trait::async_trait]
impl PaymentIntentInterface for MockDb {
#[cfg(feature = "olap")]
async fn filter_payment_intent_by_constraints(
&self,
_merchant_id: &str,
_filters: &PaymentIntentFetchConstraints,
_storage_scheme: MerchantStorageScheme,
) -> CustomResult<Vec<PaymentIntent>, StorageError> {
// [#172]: Implement function for `MockDb`
Err(StorageError::MockDbError)?
}
#[cfg(feature = "olap")]
async fn filter_payment_intents_by_time_range_constraints(
&self,
_merchant_id: &str,
_time_range: &api_models::payments::TimeRange,
_storage_scheme: MerchantStorageScheme,
) -> CustomResult<Vec<PaymentIntent>, StorageError> {
// [#172]: Implement function for `MockDb`
Err(StorageError::MockDbError)?
}
#[cfg(feature = "olap")]
async fn get_filtered_active_attempt_ids_for_total_count(
&self,
_merchant_id: &str,
_constraints: &PaymentIntentFetchConstraints,
_storage_scheme: MerchantStorageScheme,
) -> error_stack::Result<Vec<String>, StorageError> {
// [#172]: Implement function for `MockDb`
Err(StorageError::MockDbError)?
}
#[cfg(feature = "olap")]
async fn get_filtered_payment_intents_attempt(
&self,
_merchant_id: &str,
_constraints: &PaymentIntentFetchConstraints,
_storage_scheme: MerchantStorageScheme,
) -> error_stack::Result<Vec<(PaymentIntent, PaymentAttempt)>, StorageError> {
// [#172]: Implement function for `MockDb`
Err(StorageError::MockDbError)?
}
#[allow(clippy::panic)]
async fn insert_payment_intent(
&self,
new: PaymentIntentNew,
_storage_scheme: MerchantStorageScheme,
) -> CustomResult<PaymentIntent, StorageError> {
let mut payment_intents = self.payment_intents.lock().await;
let time = common_utils::date_time::now();
let payment_intent = PaymentIntent {
#[allow(clippy::as_conversions)]
id: payment_intents
.len()
.try_into()
.into_report()
.change_context(StorageError::MockDbError)?,
payment_id: new.payment_id,
merchant_id: new.merchant_id,
status: new.status,
amount: new.amount,
currency: new.currency,
amount_captured: new.amount_captured,
customer_id: new.customer_id,
description: new.description,
return_url: new.return_url,
metadata: new.metadata,
connector_id: new.connector_id,
shipping_address_id: new.shipping_address_id,
billing_address_id: new.billing_address_id,
statement_descriptor_name: new.statement_descriptor_name,
statement_descriptor_suffix: new.statement_descriptor_suffix,
created_at: new.created_at.unwrap_or(time),
modified_at: new.modified_at.unwrap_or(time),
last_synced: new.last_synced,
setup_future_usage: new.setup_future_usage,
off_session: new.off_session,
client_secret: new.client_secret,
business_country: new.business_country,
business_label: new.business_label,
active_attempt_id: new.active_attempt_id.to_owned(),
order_details: new.order_details,
allowed_payment_method_types: new.allowed_payment_method_types,
connector_metadata: new.connector_metadata,
feature_metadata: new.feature_metadata,
attempt_count: new.attempt_count,
profile_id: new.profile_id,
merchant_decision: new.merchant_decision,
};
payment_intents.push(payment_intent.clone());
Ok(payment_intent)
}
// safety: only used for testing
#[allow(clippy::unwrap_used)]
async fn update_payment_intent(
&self,
this: PaymentIntent,
update: PaymentIntentUpdate,
_storage_scheme: MerchantStorageScheme,
) -> CustomResult<PaymentIntent, StorageError> {
let mut payment_intents = self.payment_intents.lock().await;
let payment_intent = payment_intents
.iter_mut()
.find(|item| item.id == this.id)
.unwrap();
*payment_intent = update.apply_changeset(this);
Ok(payment_intent.clone())
}
// safety: only used for testing
#[allow(clippy::unwrap_used)]
async fn find_payment_intent_by_payment_id_merchant_id(
&self,
payment_id: &str,
merchant_id: &str,
_storage_scheme: MerchantStorageScheme,
) -> CustomResult<PaymentIntent, StorageError> {
let payment_intents = self.payment_intents.lock().await;
Ok(payment_intents
.iter()
.find(|payment_intent| {
payment_intent.payment_id == payment_id && payment_intent.merchant_id == merchant_id
})
.cloned()
.unwrap())
}
}
impl DataModelExt for PaymentIntentNew {
type StorageModel = DieselPaymentIntentNew;