rust/hyperswitch
1
0
Fork 0
mirror of https://github.com/juspay/hyperswitch.git synced 2025-10-30 01:27:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: add support for extending file storage to other schemes and provide a runtime flag for the same (#3348)

Browse Source 
Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com>
This commit is contained in:
Chethan Rao
2024-01-30 13:16:03 +05:30
committed by GitHub
parent 937aea906e
commit a9638d118e
18 changed files with 461 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/router/Cargo.toml
View File

@ -10,10 +10,10 @@ license.workspace = true
[features]
default = ["kv_store", "stripe", "oltp", "olap", "backwards_compatibility", "accounts_cache", "dummy_connector", "payouts", "business_profile_routing", "connector_choice_mca_id", "profile_specific_fallback_routing", "retry", "frm"]
aws_s3 = ["dep:aws-sdk-s3", "dep:aws-config"]
kms = ["external_services/kms", "dep:aws-config"]
aws_s3 = ["external_services/aws_s3"]
kms = ["external_services/kms"]
hashicorp-vault = ["external_services/hashicorp-vault"]
email = ["external_services/email", "dep:aws-config", "olap"]
email = ["external_services/email", "olap"]
frm = []
stripe = ["dep:serde_qs"]
release = ["kms", "stripe", "aws_s3", "email", "backwards_compatibility", "business_profile_routing", "accounts_cache", "kv_store", "connector_choice_mca_id", "profile_specific_fallback_routing", "vergen", "recon"]
@ -42,8 +42,6 @@ actix-web = "4.3.1"
async-bb8-diesel = { git = "https://github.com/jarnura/async-bb8-diesel", rev = "53b4ab901aab7635c8215fd1c2d542c8db443094" }
argon2 = { version = "0.5.0", features = ["std"] }
async-trait = "0.1.68"
aws-config = { version = "0.55.3", optional = true }
aws-sdk-s3 = { version = "0.28.0", optional = true }
base64 = "0.21.2"
bb8 = "0.8"
bigdecimal = "0.3.1"

21
crates/router/src/configs/settings.rs
View File

@ -11,6 +11,7 @@ use common_utils::ext_traits::ConfigExt;
use config::{Environment, File};
#[cfg(feature = "email")]
use external_services::email::EmailSettings;
use external_services::file_storage::FileStorageConfig;
#[cfg(feature = "hashicorp-vault")]
use external_services::hashicorp_vault;
#[cfg(feature = "kms")]
@ -90,10 +91,9 @@ pub struct Settings {
pub api_keys: ApiKeys,
#[cfg(feature = "kms")]
pub kms: kms::KmsConfig,
pub file_storage: FileStorageConfig,
#[cfg(feature = "hashicorp-vault")]
pub hc_vault: hashicorp_vault::HashiCorpVaultConfig,
#[cfg(feature = "aws_s3")]
pub file_upload_config: FileUploadConfig,
pub tokenization: TokenizationConfig,
pub connector_customer: ConnectorCustomer,
#[cfg(feature = "dummy_connector")]
@ -721,16 +721,6 @@ pub struct ApiKeys {
pub expiry_reminder_days: Vec<u8>,
}
#[cfg(feature = "aws_s3")]
#[derive(Debug, Deserialize, Clone, Default)]
#[serde(default)]
pub struct FileUploadConfig {
/// The AWS region to send file uploads
pub region: String,
/// The AWS s3 bucket to send file uploads
pub bucket_name: String,
}
#[derive(Debug, Deserialize, Clone, Default)]
pub struct DelayedSessionConfig {
#[serde(deserialize_with = "deser_to_get_connectors")]
@ -853,8 +843,11 @@ impl Settings {
self.kms
.validate()
.map_err(|error| ApplicationError::InvalidConfigurationValueError(error.into()))?;
#[cfg(feature = "aws_s3")]
self.file_upload_config.validate()?;
self.file_storage
.validate()
.map_err(|err| ApplicationError::InvalidConfigurationValueError(err.to_string()))?;
self.lock_settings.validate()?;
self.events.validate()?;
Ok(())

19
crates/router/src/configs/validations.rs
View File

@ -127,25 +127,6 @@ impl super::settings::DrainerSettings {
}
}
#[cfg(feature = "aws_s3")]
impl super::settings::FileUploadConfig {
pub fn validate(&self) -> Result<(), ApplicationError> {
use common_utils::fp_utils::when;
when(self.region.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"s3 region must not be empty".into(),
))
})?;
when(self.bucket_name.is_default_or_empty(), || {
Err(ApplicationError::InvalidConfigurationValueError(
"s3 bucket name must not be empty".into(),
))
})
}
}
impl super::settings::ApiKeys {
pub fn validate(&self) -> Result<(), ApplicationError> {
use common_utils::fp_utils::when;

8
crates/router/src/core/files.rs
View File

@ -1,9 +1,4 @@
pub mod helpers;
#[cfg(feature = "aws_s3")]
pub mod s3_utils;
#[cfg(not(feature = "aws_s3"))]
pub mod fs_utils;
use api_models::files;
use error_stack::{IntoReport, ResultExt};
@ -29,10 +24,7 @@ pub async fn files_create_core(
)
.await?;
let file_id = common_utils::generate_id(consts::ID_LENGTH, "file");
#[cfg(feature = "aws_s3")]
let file_key = format!("{}/{}", merchant_account.merchant_id, file_id);
#[cfg(not(feature = "aws_s3"))]
let file_key = format!("{}_{}", merchant_account.merchant_id, file_id);
let file_new = diesel_models::file::FileMetadataNew {
file_id: file_id.clone(),
merchant_id: merchant_account.merchant_id.clone(),

57
crates/router/src/core/files/fs_utils.rs
View File

@ -1,57 +0,0 @@
use std::{
fs::{remove_file, File},
io::{Read, Write},
path::PathBuf,
};
use common_utils::errors::CustomResult;
use error_stack::{IntoReport, ResultExt};
use crate::{core::errors, env};
pub fn get_file_path(file_key: String) -> PathBuf {
let mut file_path = PathBuf::new();
file_path.push(env::workspace_path());
file_path.push("files");
file_path.push(file_key);
file_path
}
pub fn save_file_to_fs(
file_key: String,
file_data: Vec<u8>,
) -> CustomResult<(), errors::ApiErrorResponse> {
let file_path = get_file_path(file_key);
let mut file = File::create(file_path)
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed to create file")?;
file.write_all(&file_data)
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed while writing into file")?;
Ok(())
}
pub fn delete_file_from_fs(file_key: String) -> CustomResult<(), errors::ApiErrorResponse> {
let file_path = get_file_path(file_key);
remove_file(file_path)
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed while deleting the file")?;
Ok(())
}
pub fn retrieve_file_from_fs(file_key: String) -> CustomResult<Vec<u8>, errors::ApiErrorResponse> {
let mut received_data: Vec<u8> = Vec::new();
let file_path = get_file_path(file_key);
let mut file = File::open(file_path)
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed while opening the file")?;
file.read_to_end(&mut received_data)
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed while reading the file")?;
Ok(received_data)
}

67
crates/router/src/core/files/helpers.rs
View File

@ -6,7 +6,7 @@ use futures::TryStreamExt;
use crate::{
core::{
errors::{self, StorageErrorExt},
files, payments, utils,
payments, utils,
},
routes::AppState,
services,
@ -30,37 +30,6 @@ pub async fn get_file_purpose(field: &mut Field) -> Option<api::FilePurpose> {
}
}
pub async fn upload_file(
#[cfg(feature = "aws_s3")] state: &AppState,
file_key: String,
file: Vec<u8>,
) -> CustomResult<(), errors::ApiErrorResponse> {
#[cfg(feature = "aws_s3")]
return files::s3_utils::upload_file_to_s3(state, file_key, file).await;
#[cfg(not(feature = "aws_s3"))]
return files::fs_utils::save_file_to_fs(file_key, file);
}
pub async fn delete_file(
#[cfg(feature = "aws_s3")] state: &AppState,
file_key: String,
) -> CustomResult<(), errors::ApiErrorResponse> {
#[cfg(feature = "aws_s3")]
return files::s3_utils::delete_file_from_s3(state, file_key).await;
#[cfg(not(feature = "aws_s3"))]
return files::fs_utils::delete_file_from_fs(file_key);
}
pub async fn retrieve_file(
#[cfg(feature = "aws_s3")] state: &AppState,
file_key: String,
) -> CustomResult<Vec<u8>, errors::ApiErrorResponse> {
#[cfg(feature = "aws_s3")]
return files::s3_utils::retrieve_file_from_s3(state, file_key).await;
#[cfg(not(feature = "aws_s3"))]
return files::fs_utils::retrieve_file_from_fs(file_key);
}
pub async fn validate_file_upload(
state: &AppState,
merchant_account: domain::MerchantAccount,
@ -132,14 +101,11 @@ pub async fn delete_file_using_file_id(
.attach_printable("File not available")?,
};
match provider {
diesel_models::enums::FileUploadProvider::Router => {
delete_file(
#[cfg(feature = "aws_s3")]
state,
provider_file_id,
)
diesel_models::enums::FileUploadProvider::Router => state
.file_storage_client
.delete_file(&provider_file_id)
.await
}
.change_context(errors::ApiErrorResponse::InternalServerError),
_ => Err(errors::ApiErrorResponse::FileProviderNotSupported {
message: "Not Supported because provider is not Router".to_string(),
}
@ -234,12 +200,11 @@ pub async fn retrieve_file_and_provider_file_id_from_file_id(
match provider {
diesel_models::enums::FileUploadProvider::Router => Ok((
Some(
retrieve_file(
#[cfg(feature = "aws_s3")]
state,
provider_file_id.clone(),
)
.await?,
state
.file_storage_client
.retrieve_file(&provider_file_id)
.await
.change_context(errors::ApiErrorResponse::InternalServerError)?,
),
Some(provider_file_id),
)),
@ -364,13 +329,11 @@ pub async fn upload_and_get_provider_provider_file_id_profile_id(
payment_attempt.merchant_connector_id,
))
} else {
upload_file(
#[cfg(feature = "aws_s3")]
state,
file_key.clone(),
create_file_request.file.clone(),
)
.await?;
state
.file_storage_client
.upload_file(&file_key, create_file_request.file.clone())
.await
.change_context(errors::ApiErrorResponse::InternalServerError)?;
Ok((
file_key,
api_models::enums::FileUploadProvider::Router,

87
crates/router/src/core/files/s3_utils.rs
View File

@ -1,87 +0,0 @@
use aws_config::{self, meta::region::RegionProviderChain};
use aws_sdk_s3::{config::Region, Client};
use common_utils::errors::CustomResult;
use error_stack::{IntoReport, ResultExt};
use futures::TryStreamExt;
use crate::{core::errors, routes};
async fn get_aws_client(state: &routes::AppState) -> Client {
let region_provider =
RegionProviderChain::first_try(Region::new(state.conf.file_upload_config.region.clone()));
let sdk_config = aws_config::from_env().region(region_provider).load().await;
Client::new(&sdk_config)
}
pub async fn upload_file_to_s3(
state: &routes::AppState,
file_key: String,
file: Vec<u8>,
) -> CustomResult<(), errors::ApiErrorResponse> {
let client = get_aws_client(state).await;
let bucket_name = &state.conf.file_upload_config.bucket_name;
// Upload file to S3
let upload_res = client
.put_object()
.bucket(bucket_name)
.key(file_key.clone())
.body(file.into())
.send()
.await;
upload_res
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("File upload to S3 failed")?;
Ok(())
}
pub async fn delete_file_from_s3(
state: &routes::AppState,
file_key: String,
) -> CustomResult<(), errors::ApiErrorResponse> {
let client = get_aws_client(state).await;
let bucket_name = &state.conf.file_upload_config.bucket_name;
// Delete file from S3
let delete_res = client
.delete_object()
.bucket(bucket_name)
.key(file_key)
.send()
.await;
delete_res
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("File delete from S3 failed")?;
Ok(())
}
pub async fn retrieve_file_from_s3(
state: &routes::AppState,
file_key: String,
) -> CustomResult<Vec<u8>, errors::ApiErrorResponse> {
let client = get_aws_client(state).await;
let bucket_name = &state.conf.file_upload_config.bucket_name;
// Get file data from S3
let get_res = client
.get_object()
.bucket(bucket_name)
.key(file_key)
.send()
.await;
let mut object = get_res
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("File retrieve from S3 failed")?;
let mut received_data: Vec<u8> = Vec::new();
while let Some(bytes) = object
.body
.try_next()
.await
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Invalid file data received from S3")?
{
received_data.extend_from_slice(&bytes); // Collect the bytes in the Vec
}
Ok(received_data)
}

5
crates/router/src/routes/app.rs
View File

@ -5,6 +5,7 @@ use actix_web::{web, Scope};
use analytics::AnalyticsConfig;
#[cfg(feature = "email")]
use external_services::email::{ses::AwsSes, EmailService};
use external_services::file_storage::FileStorageInterface;
#[cfg(all(feature = "olap", feature = "hashicorp-vault"))]
use external_services::hashicorp_vault::decrypt::VaultFetch;
#[cfg(feature = "kms")]
@ -68,6 +69,7 @@ pub struct AppState {
#[cfg(feature = "olap")]
pub pool: crate::analytics::AnalyticsProvider,
pub request_id: Option<RequestId>,
pub file_storage_client: Box<dyn FileStorageInterface>,
}
impl scheduler::SchedulerAppState for AppState {
@ -266,6 +268,8 @@ impl AppState {
#[cfg(feature = "email")]
let email_client = Arc::new(create_email_client(&conf).await);
let file_storage_client = conf.file_storage.get_file_storage_client().await;
Self {
flow_name: String::from("default"),
store,
@ -279,6 +283,7 @@ impl AppState {
#[cfg(feature = "olap")]
pool,
request_id: None,
file_storage_client,
}
})
.await