rust/hyperswitch
1
0
Fork 0
mirror of https://github.com/juspay/hyperswitch.git synced 2025-10-27 19:46:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(drainer): added drainer which reads from redis stream and executes queries on DB (#142)

Browse Source
This commit is contained in:
Abhishek
2022-12-16 15:38:03 +05:30
committed by GitHub
parent 3db49d0530
commit 3bad58b0d3
41 changed files with 648 additions and 655 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/router/Cargo.toml
View File

@ -75,7 +75,7 @@ masking = { version = "0.1.0", path = "../masking" }
redis_interface = { version = "0.1.0", path = "../redis_interface" }
router_derive = { version = "0.1.0", path = "../router_derive" }
router_env = { version = "0.1.0", path = "../router_env", features = ["log_extra_implicit_fields", "log_custom_entries_to_extra"] }
storage_models = { version = "0.1.0", path = "../storage_models" }
storage_models = { version = "0.1.0", path = "../storage_models", features = ["kv_store"] }
[build-dependencies]
router_env = { version = "0.1.0", path = "../router_env", default-features = false, features = ["vergen"] }

7
crates/router/src/configs/defaults.toml
View File

@ -53,8 +53,9 @@ batch_size = 200
[drainer]
stream_name = "DRAINER_STREAM"
num_partitions = 4
num_partitions = 64
max_read_count = 100
[connectors.supported]
wallets = ["klarna","braintree"]
cards = ["stripe","adyen","authorizedotnet","checkout","braintree"]
wallets = ["klarna", "braintree"]
cards = ["stripe", "adyen", "authorizedotnet", "checkout", "braintree"]

1
crates/router/src/configs/settings.rs
View File

@ -145,6 +145,7 @@ pub struct ProducerSettings {
pub struct DrainerSettings {
pub stream_name: String,
pub num_partitions: u8,
pub max_read_count: u64,
}
impl Settings {

36
crates/router/src/db/payment_attempt.rs
View File

@ -314,7 +314,7 @@ mod storage {
connection::pg_connection,
core::errors::{self, CustomResult},
services::Store,
types::storage::{enums, payment_attempt::*},
types::storage::{enums, kv, payment_attempt::*},
utils::storage_partitioning::KvStorePartition,
};
@ -386,11 +386,11 @@ mod storage {
))
.into_report(),
Ok(HsetnxReply::KeySet) => {
let conn = pg_connection(&self.master_pool).await;
let query = payment_attempt
.insert_query(&conn)
.await
.change_context(errors::StorageError::KVError)?;
let redis_entry = kv::TypedSql {
op: kv::DBOperation::Insert {
insertable: kv::Insertable::PaymentAttempt(payment_attempt),
},
};
let stream_name = self.drainer_stream(&PaymentAttempt::shard_key(
crate::utils::storage_partitioning::PartitionKey::MerchantIdPaymentId {
merchant_id: &created_attempt.merchant_id,
@ -402,7 +402,9 @@ mod storage {
.stream_append_entry(
&stream_name,
&RedisEntryId::AutoGeneratedID,
query.to_field_value_pairs(),
redis_entry
.to_field_value_pairs()
.change_context(errors::StorageError::KVError)?,
)
.await
.change_context(errors::StorageError::KVError)?;
@ -444,11 +446,17 @@ mod storage {
.map(|_| updated_attempt)
.change_context(errors::StorageError::KVError)?;
let conn = pg_connection(&self.master_pool).await;
let query = this
.update_query(&conn, payment_attempt)
.await
.change_context(errors::StorageError::KVError)?;
let redis_entry = kv::TypedSql {
op: kv::DBOperation::Update {
updatable: kv::Updateable::PaymentAttemptUpdate(
kv::PaymentAttemptUpdateMems {
orig: this,
update_data: payment_attempt,
},
),
},
};
let stream_name = self.drainer_stream(&PaymentAttempt::shard_key(
crate::utils::storage_partitioning::PartitionKey::MerchantIdPaymentId {
merchant_id: &updated_attempt.merchant_id,
@ -460,7 +468,9 @@ mod storage {
.stream_append_entry(
&stream_name,
&RedisEntryId::AutoGeneratedID,
query.to_field_value_pairs(),
redis_entry
.to_field_value_pairs()
.change_context(errors::StorageError::KVError)?,
)
.await
.change_context(errors::StorageError::KVError)?;

36
crates/router/src/db/payment_intent.rs
View File

@ -50,7 +50,7 @@ mod storage {
services::Store,
types::{
api,
storage::{enums, payment_intent::*},
storage::{enums, kv, payment_intent::*},
},
utils::storage_partitioning::KvStorePartition,
};
@ -105,11 +105,11 @@ mod storage {
))
.into_report(),
Ok(HsetnxReply::KeySet) => {
let conn = pg_connection(&self.master_pool).await;
let query = new
.insert_query(&conn)
.await
.change_context(errors::StorageError::KVError)?;
let redis_entry = kv::TypedSql {
op: kv::DBOperation::Insert {
insertable: kv::Insertable::PaymentIntent(new),
},
};
let stream_name = self.drainer_stream(&PaymentIntent::shard_key(
crate::utils::storage_partitioning::PartitionKey::MerchantIdPaymentId {
merchant_id: &created_intent.merchant_id,
@ -121,7 +121,9 @@ mod storage {
.stream_append_entry(
&stream_name,
&RedisEntryId::AutoGeneratedID,
query.to_field_value_pairs(),
redis_entry
.to_field_value_pairs()
.change_context(errors::StorageError::KVError)?,
)
.await
.change_context(errors::StorageError::KVError)?;
@ -163,11 +165,17 @@ mod storage {
.map(|_| updated_intent)
.change_context(errors::StorageError::KVError)?;
let conn = pg_connection(&self.master_pool).await;
let query = this
.update_query(&conn, payment_intent)
.await
.change_context(errors::StorageError::KVError)?;
let redis_entry = kv::TypedSql {
op: kv::DBOperation::Update {
updatable: kv::Updateable::PaymentIntentUpdate(
kv::PaymentIntentUpdateMems {
orig: this,
update_data: payment_intent,
},
),
},
};
let stream_name = self.drainer_stream(&PaymentIntent::shard_key(
crate::utils::storage_partitioning::PartitionKey::MerchantIdPaymentId {
merchant_id: &updated_intent.merchant_id,
@ -179,7 +187,9 @@ mod storage {
.stream_append_entry(
&stream_name,
&RedisEntryId::AutoGeneratedID,
query.to_field_value_pairs(),
redis_entry
.to_field_value_pairs()
.change_context(errors::StorageError::KVError)?,
)
.await
.change_context(errors::StorageError::KVError)?;

2
crates/router/src/services.rs
View File

@ -41,7 +41,7 @@ impl Store {
#[cfg(feature = "kv_store")]
pub fn drainer_stream(&self, shard_key: &str) -> String {
// "{shard_key}_stream_name"
// Example: {shard_5}_drainer_stream
format!("{{{}}}_{}", shard_key, self.config.drainer_stream_name,)
}
}

3
crates/router/src/types/storage.rs
View File

@ -18,6 +18,9 @@ mod query;
pub mod refund;
pub mod temp_card;
#[cfg(feature = "kv_store")]
pub mod kv;
pub use self::{
address::*, configs::*, connector_response::*, customers::*, events::*, locker_mock_up::*,
mandate::*, merchant_account::*, merchant_connector_account::*, payment_attempt::*,

4
crates/router/src/types/storage/kv.rs Normal file
View File

@ -0,0 +1,4 @@
pub use storage_models::kv::{
DBOperation, Insertable, PaymentAttemptUpdateMems, PaymentIntentUpdateMems, TypedSql,
Updateable,
};

1
crates/router/src/types/storage/payment_intent.rs
View File

@ -38,7 +38,6 @@ impl PaymentIntentDbExt for PaymentIntent {
//TODO: Replace this with Boxable Expression and pass it into generic filter
// when https://github.com/rust-lang/rust/issues/52662 becomes stable
let mut filter = <PaymentIntent as HasTable>::table()
.filter(dsl::merchant_id.eq(merchant_id.to_owned()))
.order_by(dsl::id)