feat: encryption service integration to support batch encryption and decryption (#5164)

Co-authored-by: dracarys18 <karthikey.hegde@juspay.in>
Co-authored-by: hyperswitch-bot[bot] <148525504+hyperswitch-bot[bot]@users.noreply.github.com>
This commit is contained in:
Arjun Karthik
2024-07-19 13:08:58 +05:30
committed by GitHub
parent c698921c41
commit 33298b3808
127 changed files with 4239 additions and 1378 deletions

View File

@ -10,6 +10,7 @@ license.workspace = true
[features]
keymanager = ["dep:router_env"]
keymanager_mtls = ["reqwest/rustls-tls"]
encryption_service = ["dep:router_env"]
signals = ["dep:signal-hook-tokio", "dep:signal-hook", "dep:tokio", "dep:router_env", "dep:futures"]
async_ext = ["dep:async-trait", "dep:futures"]
logs = ["dep:router_env"]
@ -17,6 +18,7 @@ metrics = ["dep:router_env", "dep:futures"]
[dependencies]
async-trait = { version = "0.1.79", optional = true }
base64 = "0.22.0"
blake3 = { version = "1.5.1", features = ["serde"] }
bytes = "1.6.0"
diesel = "2.1.5"

View File

@ -99,6 +99,8 @@ pub const MAX_ALLOWED_MERCHANT_REFERENCE_ID_LENGTH: u8 = 64;
/// Minimum allowed length for MerchantReferenceId
pub const MIN_REQUIRED_MERCHANT_REFERENCE_ID_LENGTH: u8 = 1;
/// General purpose base64 engine
pub const BASE64_ENGINE: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD;
/// Regex for matching a domain
/// Eg -
/// http://www.example.com

View File

@ -0,0 +1,73 @@
use diesel::{
backend::Backend,
deserialize::{self, FromSql, Queryable},
expression::AsExpression,
serialize::ToSql,
sql_types,
};
use masking::Secret;
use crate::{crypto::Encryptable, pii::EncryptionStrategy};
impl<DB> FromSql<sql_types::Binary, DB> for Encryption
where
DB: Backend,
Secret<Vec<u8>, EncryptionStrategy>: FromSql<sql_types::Binary, DB>,
{
fn from_sql(bytes: DB::RawValue<'_>) -> deserialize::Result<Self> {
<Secret<Vec<u8>, EncryptionStrategy>>::from_sql(bytes).map(Self::new)
}
}
impl<DB> ToSql<sql_types::Binary, DB> for Encryption
where
DB: Backend,
Secret<Vec<u8>, EncryptionStrategy>: ToSql<sql_types::Binary, DB>,
{
fn to_sql<'b>(
&'b self,
out: &mut diesel::serialize::Output<'b, '_, DB>,
) -> diesel::serialize::Result {
self.get_inner().to_sql(out)
}
}
impl<DB> Queryable<sql_types::Binary, DB> for Encryption
where
DB: Backend,
Secret<Vec<u8>, EncryptionStrategy>: FromSql<sql_types::Binary, DB>,
{
type Row = Secret<Vec<u8>, EncryptionStrategy>;
fn build(row: Self::Row) -> deserialize::Result<Self> {
Ok(Self { inner: row })
}
}
#[derive(Debug, AsExpression, Clone, serde::Serialize, serde::Deserialize, Eq, PartialEq)]
#[diesel(sql_type = sql_types::Binary)]
#[repr(transparent)]
pub struct Encryption {
inner: Secret<Vec<u8>, EncryptionStrategy>,
}
impl<T: Clone> From<Encryptable<T>> for Encryption {
fn from(value: Encryptable<T>) -> Self {
Self::new(value.into_encrypted())
}
}
impl Encryption {
pub fn new(item: Secret<Vec<u8>, EncryptionStrategy>) -> Self {
Self { inner: item }
}
#[inline]
pub fn into_inner(self) -> Secret<Vec<u8>, EncryptionStrategy> {
self.inner
}
#[inline]
pub fn get_inner(&self) -> &Secret<Vec<u8>, EncryptionStrategy> {
&self.inner
}
}

View File

@ -3,22 +3,26 @@
use core::fmt::Debug;
use std::str::FromStr;
use base64::Engine;
use error_stack::ResultExt;
use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode};
#[cfg(feature = "keymanager_mtls")]
use masking::PeekInterface;
use masking::{PeekInterface, StrongSecret};
use once_cell::sync::OnceCell;
use router_env::{instrument, logger, tracing};
use crate::{
consts::BASE64_ENGINE,
errors,
types::keymanager::{
DataKeyCreateResponse, EncryptionCreateRequest, EncryptionTransferRequest, KeyManagerState,
BatchDecryptDataRequest, DataKeyCreateResponse, DecryptDataRequest,
EncryptionCreateRequest, EncryptionTransferRequest, KeyManagerState,
TransientBatchDecryptDataRequest, TransientDecryptDataRequest,
},
};
const CONTENT_TYPE: &str = "Content-Type";
static ENCRYPTION_API_CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
static DEFAULT_ENCRYPTION_VERSION: &str = "v1";
/// Get keymanager client constructed from the url and state
#[instrument(skip_all)]
@ -68,7 +72,7 @@ pub async fn send_encryption_request<T>(
request_body: T,
) -> errors::CustomResult<reqwest::Response, errors::KeyManagerClientError>
where
T: serde::Serialize,
T: ConvertRaw,
{
let client = get_api_encryption_client(state)?;
let url = reqwest::Url::parse(&url)
@ -76,7 +80,7 @@ where
client
.request(method, url)
.json(&request_body)
.json(&ConvertRaw::convert_raw(request_body)?)
.headers(headers)
.send()
.await
@ -94,7 +98,7 @@ pub async fn call_encryption_service<T, R>(
request_body: T,
) -> errors::CustomResult<R, errors::KeyManagerClientError>
where
T: serde::Serialize + Send + Sync + 'static + Debug,
T: ConvertRaw + Send + Sync + 'static + Debug,
R: serde::de::DeserializeOwned,
{
let url = format!("{}/{endpoint}", &state.url);
@ -152,6 +156,62 @@ where
}
}
/// Trait to convert the raw data to the required format for encryption service request
pub trait ConvertRaw {
/// Return type of the convert_raw function
type Output: serde::Serialize;
/// Function to convert the raw data to the required format for encryption service request
fn convert_raw(self) -> Result<Self::Output, errors::KeyManagerClientError>;
}
impl<T: serde::Serialize> ConvertRaw for T {
type Output = T;
fn convert_raw(self) -> Result<Self::Output, errors::KeyManagerClientError> {
Ok(self)
}
}
impl ConvertRaw for TransientDecryptDataRequest {
type Output = DecryptDataRequest;
fn convert_raw(self) -> Result<Self::Output, errors::KeyManagerClientError> {
let data = match String::from_utf8(self.data.peek().clone()) {
Ok(data) => data,
Err(_) => {
let data = BASE64_ENGINE.encode(self.data.peek().clone());
format!("{DEFAULT_ENCRYPTION_VERSION}:{data}")
}
};
Ok(DecryptDataRequest {
identifier: self.identifier,
data: StrongSecret::new(data),
})
}
}
impl ConvertRaw for TransientBatchDecryptDataRequest {
type Output = BatchDecryptDataRequest;
fn convert_raw(self) -> Result<Self::Output, errors::KeyManagerClientError> {
let data = self
.data
.iter()
.map(|(k, v)| {
let value = match String::from_utf8(v.peek().clone()) {
Ok(data) => data,
Err(_) => {
let data = BASE64_ENGINE.encode(v.peek().clone());
format!("{DEFAULT_ENCRYPTION_VERSION}:{data}")
}
};
(k.to_owned(), StrongSecret::new(value))
})
.collect();
Ok(BatchDecryptDataRequest {
data,
identifier: self.identifier,
})
}
}
/// A function to create the key in keymanager
#[instrument(skip_all)]
pub async fn create_key_in_key_manager(

View File

@ -13,6 +13,8 @@ pub mod access_token;
pub mod consts;
pub mod crypto;
pub mod custom_serde;
#[allow(missing_docs)] // Todo: add docs
pub mod encryption;
pub mod errors;
#[allow(missing_docs)] // Todo: add docs
pub mod events;
@ -31,6 +33,7 @@ pub mod request;
pub mod signals;
#[allow(missing_docs)] // Todo: add docs
pub mod static_cache;
pub mod transformers;
pub mod types;
pub mod validation;

View File

@ -0,0 +1,15 @@
//! Utilities for converting between foreign types
/// Trait for converting from one foreign type to another
pub trait ForeignFrom<F> {
/// Convert from a foreign type to the current type
fn foreign_from(from: F) -> Self;
}
/// Trait for converting from one foreign type to another
pub trait ForeignTryFrom<F>: Sized {
/// Custom error for conversion failure
type Error;
/// Convert from a foreign type to the current type and return an error if the conversion fails
fn foreign_try_from(from: F) -> Result<Self, Self::Error>;
}

View File

@ -1,8 +1,24 @@
#![allow(missing_docs)]
#[cfg(feature = "keymanager_mtls")]
use masking::Secret;
use serde::{Deserialize, Serialize};
use core::fmt;
use base64::Engine;
use masking::{ExposeInterface, PeekInterface, Secret, Strategy, StrongSecret};
#[cfg(feature = "encryption_service")]
use router_env::logger;
use rustc_hash::FxHashMap;
use serde::{
de::{self, Unexpected, Visitor},
ser, Deserialize, Deserializer, Serialize,
};
use crate::{
consts::BASE64_ENGINE,
crypto::Encryptable,
encryption::Encryption,
errors::{self, CustomResult},
transformers::{ForeignFrom, ForeignTryFrom},
};
#[derive(Debug)]
pub struct KeyManagerState {
@ -18,6 +34,7 @@ pub struct KeyManagerState {
pub enum Identifier {
User(String),
Merchant(String),
UserAuth(String),
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
@ -39,3 +56,420 @@ pub struct DataKeyCreateResponse {
pub identifier: Identifier,
pub key_version: String,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct BatchEncryptDataRequest {
#[serde(flatten)]
pub identifier: Identifier,
pub data: DecryptedDataGroup,
}
impl<S> From<(Secret<Vec<u8>, S>, Identifier)> for EncryptDataRequest
where
S: Strategy<Vec<u8>>,
{
fn from((secret, identifier): (Secret<Vec<u8>, S>, Identifier)) -> Self {
Self {
identifier,
data: DecryptedData(StrongSecret::new(secret.expose())),
}
}
}
impl<S> From<(FxHashMap<String, Secret<Vec<u8>, S>>, Identifier)> for BatchEncryptDataRequest
where
S: Strategy<Vec<u8>>,
{
fn from((map, identifier): (FxHashMap<String, Secret<Vec<u8>, S>>, Identifier)) -> Self {
let group = map
.into_iter()
.map(|(key, value)| (key, DecryptedData(StrongSecret::new(value.expose()))))
.collect();
Self {
identifier,
data: DecryptedDataGroup(group),
}
}
}
impl<S> From<(Secret<String, S>, Identifier)> for EncryptDataRequest
where
S: Strategy<String>,
{
fn from((secret, identifier): (Secret<String, S>, Identifier)) -> Self {
Self {
data: DecryptedData(StrongSecret::new(secret.expose().as_bytes().to_vec())),
identifier,
}
}
}
impl<S> From<(Secret<serde_json::Value, S>, Identifier)> for EncryptDataRequest
where
S: Strategy<serde_json::Value>,
{
fn from((secret, identifier): (Secret<serde_json::Value, S>, Identifier)) -> Self {
Self {
data: DecryptedData(StrongSecret::new(
secret.expose().to_string().as_bytes().to_vec(),
)),
identifier,
}
}
}
impl<S> From<(FxHashMap<String, Secret<serde_json::Value, S>>, Identifier)>
for BatchEncryptDataRequest
where
S: Strategy<serde_json::Value>,
{
fn from(
(map, identifier): (FxHashMap<String, Secret<serde_json::Value, S>>, Identifier),
) -> Self {
let group = map
.into_iter()
.map(|(key, value)| {
(
key,
DecryptedData(StrongSecret::new(
value.expose().to_string().as_bytes().to_vec(),
)),
)
})
.collect();
Self {
data: DecryptedDataGroup(group),
identifier,
}
}
}
impl<S> From<(FxHashMap<String, Secret<String, S>>, Identifier)> for BatchEncryptDataRequest
where
S: Strategy<String>,
{
fn from((map, identifier): (FxHashMap<String, Secret<String, S>>, Identifier)) -> Self {
let group = map
.into_iter()
.map(|(key, value)| {
(
key,
DecryptedData(StrongSecret::new(value.expose().as_bytes().to_vec())),
)
})
.collect();
Self {
data: DecryptedDataGroup(group),
identifier,
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct EncryptDataRequest {
#[serde(flatten)]
pub identifier: Identifier,
pub data: DecryptedData,
}
#[derive(Debug, Serialize, serde::Deserialize)]
pub struct DecryptedDataGroup(pub FxHashMap<String, DecryptedData>);
#[derive(Debug, Serialize, Deserialize)]
pub struct BatchEncryptDataResponse {
pub data: EncryptedDataGroup,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct EncryptDataResponse {
pub data: EncryptedData,
}
#[derive(Debug, Serialize, serde::Deserialize)]
pub struct EncryptedDataGroup(pub FxHashMap<String, EncryptedData>);
#[derive(Debug)]
pub struct TransientBatchDecryptDataRequest {
pub identifier: Identifier,
pub data: FxHashMap<String, StrongSecret<Vec<u8>>>,
}
#[derive(Debug)]
pub struct TransientDecryptDataRequest {
pub identifier: Identifier,
pub data: StrongSecret<Vec<u8>>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct BatchDecryptDataRequest {
#[serde(flatten)]
pub identifier: Identifier,
pub data: FxHashMap<String, StrongSecret<String>>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct DecryptDataRequest {
#[serde(flatten)]
pub identifier: Identifier,
pub data: StrongSecret<String>,
}
impl<T, S> ForeignFrom<(FxHashMap<String, Secret<T, S>>, BatchEncryptDataResponse)>
for FxHashMap<String, Encryptable<Secret<T, S>>>
where
T: Clone,
S: Strategy<T> + Send,
{
fn foreign_from(
(mut masked_data, response): (FxHashMap<String, Secret<T, S>>, BatchEncryptDataResponse),
) -> Self {
response
.data
.0
.into_iter()
.flat_map(|(k, v)| {
masked_data.remove(&k).map(|inner| {
(
k,
Encryptable::new(inner.clone(), v.data.peek().clone().into()),
)
})
})
.collect()
}
}
impl<T, S> ForeignFrom<(Secret<T, S>, EncryptDataResponse)> for Encryptable<Secret<T, S>>
where
T: Clone,
S: Strategy<T> + Send,
{
fn foreign_from((masked_data, response): (Secret<T, S>, EncryptDataResponse)) -> Self {
Self::new(masked_data, response.data.data.peek().clone().into())
}
}
pub trait DecryptedDataConversion<T: Clone, S: Strategy<T> + Send>: Sized {
fn convert(
value: &DecryptedData,
encryption: Encryption,
) -> CustomResult<Self, errors::CryptoError>;
}
impl<S: Strategy<String> + Send> DecryptedDataConversion<String, S>
for Encryptable<Secret<String, S>>
{
fn convert(
value: &DecryptedData,
encryption: Encryption,
) -> CustomResult<Self, errors::CryptoError> {
let string = String::from_utf8(value.clone().inner().peek().clone()).map_err(|_err| {
#[cfg(feature = "encryption_service")]
logger::error!("Decryption error {:?}", _err);
errors::CryptoError::DecodingFailed
})?;
Ok(Self::new(Secret::new(string), encryption.into_inner()))
}
}
impl<S: Strategy<serde_json::Value> + Send> DecryptedDataConversion<serde_json::Value, S>
for Encryptable<Secret<serde_json::Value, S>>
{
fn convert(
value: &DecryptedData,
encryption: Encryption,
) -> CustomResult<Self, errors::CryptoError> {
let val = serde_json::from_slice(value.clone().inner().peek()).map_err(|_err| {
#[cfg(feature = "encryption_service")]
logger::error!("Decryption error {:?}", _err);
errors::CryptoError::DecodingFailed
})?;
Ok(Self::new(Secret::new(val), encryption.clone().into_inner()))
}
}
impl<S: Strategy<Vec<u8>> + Send> DecryptedDataConversion<Vec<u8>, S>
for Encryptable<Secret<Vec<u8>, S>>
{
fn convert(
value: &DecryptedData,
encryption: Encryption,
) -> CustomResult<Self, errors::CryptoError> {
Ok(Self::new(
Secret::new(value.clone().inner().peek().clone()),
encryption.clone().into_inner(),
))
}
}
impl<T, S> ForeignTryFrom<(Encryption, DecryptDataResponse)> for Encryptable<Secret<T, S>>
where
T: Clone,
S: Strategy<T> + Send,
Self: DecryptedDataConversion<T, S>,
{
type Error = error_stack::Report<errors::CryptoError>;
fn foreign_try_from(
(encrypted_data, response): (Encryption, DecryptDataResponse),
) -> Result<Self, Self::Error> {
Self::convert(&response.data, encrypted_data)
}
}
impl<T, S> ForeignTryFrom<(FxHashMap<String, Encryption>, BatchDecryptDataResponse)>
for FxHashMap<String, Encryptable<Secret<T, S>>>
where
T: Clone,
S: Strategy<T> + Send,
Encryptable<Secret<T, S>>: DecryptedDataConversion<T, S>,
{
type Error = error_stack::Report<errors::CryptoError>;
fn foreign_try_from(
(mut encrypted_data, response): (FxHashMap<String, Encryption>, BatchDecryptDataResponse),
) -> Result<Self, Self::Error> {
response
.data
.0
.into_iter()
.map(|(k, v)| match encrypted_data.remove(&k) {
Some(encrypted) => Ok((k.clone(), Encryptable::convert(&v, encrypted.clone())?)),
None => Err(errors::CryptoError::DecodingFailed)?,
})
.collect()
}
}
impl From<(Encryption, Identifier)> for TransientDecryptDataRequest {
fn from((encryption, identifier): (Encryption, Identifier)) -> Self {
Self {
data: StrongSecret::new(encryption.clone().into_inner().expose()),
identifier,
}
}
}
impl From<(FxHashMap<String, Encryption>, Identifier)> for TransientBatchDecryptDataRequest {
fn from((map, identifier): (FxHashMap<String, Encryption>, Identifier)) -> Self {
let data = map
.into_iter()
.map(|(k, v)| (k, StrongSecret::new(v.clone().into_inner().expose())))
.collect();
Self { data, identifier }
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct BatchDecryptDataResponse {
pub data: DecryptedDataGroup,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct DecryptDataResponse {
pub data: DecryptedData,
}
#[derive(Clone, Debug)]
pub struct DecryptedData(StrongSecret<Vec<u8>>);
impl Serialize for DecryptedData {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let data = BASE64_ENGINE.encode(self.0.peek());
serializer.serialize_str(&data)
}
}
impl<'de> Deserialize<'de> for DecryptedData {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct DecryptedDataVisitor;
impl<'de> Visitor<'de> for DecryptedDataVisitor {
type Value = DecryptedData;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("string of the format {version}:{base64_encoded_data}'")
}
fn visit_str<E>(self, value: &str) -> Result<DecryptedData, E>
where
E: de::Error,
{
let dec_data = BASE64_ENGINE.decode(value).map_err(|err| {
let err = err.to_string();
E::invalid_value(Unexpected::Str(value), &err.as_str())
})?;
Ok(DecryptedData(dec_data.into()))
}
}
deserializer.deserialize_str(DecryptedDataVisitor)
}
}
impl DecryptedData {
pub fn from_data(data: StrongSecret<Vec<u8>>) -> Self {
Self(data)
}
pub fn inner(self) -> StrongSecret<Vec<u8>> {
self.0
}
}
#[derive(Debug)]
pub struct EncryptedData {
pub data: StrongSecret<Vec<u8>>,
}
impl Serialize for EncryptedData {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let data = String::from_utf8(self.data.peek().clone()).map_err(ser::Error::custom)?;
serializer.serialize_str(data.as_str())
}
}
impl<'de> Deserialize<'de> for EncryptedData {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
struct EncryptedDataVisitor;
impl<'de> Visitor<'de> for EncryptedDataVisitor {
type Value = EncryptedData;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("string of the format {version}:{base64_encoded_data}'")
}
fn visit_str<E>(self, value: &str) -> Result<EncryptedData, E>
where
E: de::Error,
{
Ok(EncryptedData {
data: StrongSecret::new(value.as_bytes().to_vec()),
})
}
}
deserializer.deserialize_str(EncryptedDataVisitor)
}
}
/// A trait which converts the struct to Hashmap required for encryption and back to struct
pub trait ToEncryptable<T, S: Clone, E>: Sized {
/// Serializes the type to a hashmap
fn to_encryptable(self) -> FxHashMap<String, E>;
/// Deserializes the hashmap back to the type
fn from_encryptable(
hashmap: FxHashMap<String, Encryptable<S>>,
) -> CustomResult<T, errors::ParsingError>;
}