refactor(proxy): specify hosts for proxy exclusion instead of complete URLs (#6957)

config/config.example.toml

@@ -23,11 +23,10 @@ certificate = "/path/to/certificate.pem"
 # Proxy server configuration for connecting to payment gateways.
 # Don't define the fields if a Proxy isn't needed. Empty strings will cause failure.
 [proxy]
-# http_url = "http proxy url"   # Proxy all HTTP traffic via this proxy
+# http_url = "http proxy url"                   # Proxy all HTTP traffic via this proxy
-# https_url = "https proxy url" # Proxy all HTTPS traffic via this proxy
+# https_url = "https proxy url"                 # Proxy all HTTPS traffic via this proxy
-idle_pool_connection_timeout = 90 # Timeout for idle pool connections (defaults to 90s)
+idle_pool_connection_timeout = 90               # Timeout for idle pool connections (defaults to 90s)
-bypass_proxy_urls = []            # A list of URLs that should bypass the proxy
+bypass_proxy_hosts = "localhost, cluster.local" # A comma-separated list of domains or IP addresses that should not use the proxy. Whitespace between entries would be ignored.
-
 
 # Configuration for the Key Manager Service
 [key_manager]
@@ -468,8 +467,8 @@ bank_redirect.giropay = { connector_list = "adyen,globalpay" }
 
 
 [mandates.update_mandate_supported]
-card.credit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card 
+card.credit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card
-card.debit = { connector_list = "cybersource" }  # Update Mandate supported payment method type and connector for card 
+card.debit = { connector_list = "cybersource" }  # Update Mandate supported payment method type and connector for card
 
 # Required fields info used while listing the payment_method_data
 [required_fields.pay_later] # payment_method = "pay_later"

config/deployments/env_specific.toml

@@ -195,9 +195,9 @@ pm_auth_key = "pm_auth_key" # Payment method auth key used for authorization
 redis_expiry = 900          # Redis expiry time in milliseconds
 
 [proxy]
-http_url = "http://proxy_http_url"    # Outgoing proxy http URL to proxy the HTTP traffic
+http_url = "http://proxy_http_url"              # Proxy all HTTP traffic via this proxy
-https_url = "https://proxy_https_url" # Outgoing proxy https URL to proxy the HTTPS traffic
+https_url = "https://proxy_https_url"           # Proxy all HTTPS traffic via this proxy
-bypass_proxy_urls = []                # A list of URLs that should bypass the proxy
+bypass_proxy_hosts = "localhost, cluster.local" # A comma-separated list of domains or IP addresses that should not use the proxy. Whitespace between entries would be ignored.
 
 # Redis credentials
 [redis]
@@ -306,8 +306,8 @@ enabled = false
 global_tenant = { tenant_id = "global", schema = "public", redis_key_prefix = "", clickhouse_database = "default"}
 
 [multitenancy.tenants.public]
-base_url = "http://localhost:8080" 
+base_url = "http://localhost:8080"
-schema = "public"     
+schema = "public"
 redis_key_prefix = ""
 clickhouse_database = "default"
 

crates/common_enums/src/enums.rs

@@ -63,6 +63,8 @@ pub enum ApiClientError {
     #[error("Unexpected state reached/Invariants conflicted")]
     UnexpectedState,
 
+    #[error("Failed to parse URL")]
+    UrlParsingFailed,
     #[error("URL encoding of request payload failed")]
     UrlEncodingFailed,
     #[error("Failed to send request to connector {0}")]

crates/router/src/bin/scheduler.rs

@@ -35,15 +35,8 @@ async fn main() -> CustomResult<(), ProcessTrackerError> {
     let conf = Settings::with_config_path(cmd_line.config_path)
         .expect("Unable to construct application configuration");
     let api_client = Box::new(
-        services::ProxyClient::new(
+        services::ProxyClient::new(&conf.proxy)
-            conf.proxy.clone(),
+            .change_context(ProcessTrackerError::ConfigurationError)?,
-            services::proxy_bypass_urls(
-                conf.key_manager.get_inner(),
-                &conf.locker,
-                &conf.proxy.bypass_proxy_urls,
-            ),
-        )
-        .change_context(ProcessTrackerError::ConfigurationError)?,
     );
     // channel for listening to redis disconnect events
     let (redis_shutdown_signal_tx, redis_shutdown_signal_rx) = oneshot::channel();

crates/router/src/configs/defaults.rs

@@ -59,7 +59,7 @@ impl Default for super::settings::Proxy {
             http_url: Default::default(),
             https_url: Default::default(),
             idle_pool_connection_timeout: Some(90),
-            bypass_proxy_urls: Vec::new(),
+            bypass_proxy_hosts: Default::default(),
         }
     }
 }

crates/router/src/configs/settings.rs

@@ -638,7 +638,7 @@ pub struct Proxy {
     pub http_url: Option<String>,
     pub https_url: Option<String>,
     pub idle_pool_connection_timeout: Option<u64>,
-    pub bypass_proxy_urls: Vec<String>,
+    pub bypass_proxy_hosts: Option<String>,
 }
 
 #[derive(Debug, Deserialize, Clone)]
@@ -828,7 +828,6 @@ impl Settings<SecuredSecret> {
                     .with_list_parse_key("log.telemetry.route_to_trace")
                     .with_list_parse_key("redis.cluster_urls")
                     .with_list_parse_key("events.kafka.brokers")
-                    .with_list_parse_key("proxy.bypass_proxy_urls")
                     .with_list_parse_key("connectors.supported.wallets")
                     .with_list_parse_key("connector_request_reference_id_config.merchant_ids_send_payment_id_as_connector_request_id"),
 

crates/router/src/lib.rs

@@ -237,19 +237,9 @@ pub async fn start_server(conf: settings::Settings<SecuredSecret>) -> Applicatio
     logger::debug!(startup_config=?conf);
     let server = conf.server.clone();
     let (tx, rx) = oneshot::channel();
-    let api_client = Box::new(
+    let api_client = Box::new(services::ProxyClient::new(&conf.proxy).map_err(|error| {
-        services::ProxyClient::new(
+        errors::ApplicationError::ApiClientError(error.current_context().clone())
-            conf.proxy.clone(),
+    })?);
-            services::proxy_bypass_urls(
-                conf.key_manager.get_inner(),
-                &conf.locker,
-                &conf.proxy.bypass_proxy_urls,
-            ),
-        )
-        .map_err(|error| {
-            errors::ApplicationError::ApiClientError(error.current_context().clone())
-        })?,
-    );
     let state = Box::pin(AppState::new(conf, tx, api_client)).await;
     let request_body_limit = server.request_body_limit;
 

crates/router/src/services/api.rs

@@ -17,7 +17,7 @@ use actix_web::{
     http::header::{HeaderName, HeaderValue},
     web, FromRequest, HttpRequest, HttpResponse, Responder, ResponseError,
 };
-pub use client::{proxy_bypass_urls, ApiClient, MockApiClient, ProxyClient};
+pub use client::{ApiClient, MockApiClient, ProxyClient};
 pub use common_enums::enums::PaymentAction;
 pub use common_utils::request::{ContentType, Method, Request, RequestBuilder};
 use common_utils::{
@@ -416,29 +416,11 @@ pub async fn send_request(
 ) -> CustomResult<reqwest::Response, errors::ApiClientError> {
     logger::info!(method=?request.method, headers=?request.headers, payload=?request.body, ?request);
 
-    let url = reqwest::Url::parse(&request.url)
+    let url =
-        .change_context(errors::ApiClientError::UrlEncodingFailed)?;
+        url::Url::parse(&request.url).change_context(errors::ApiClientError::UrlParsingFailed)?;
 
-    #[cfg(feature = "dummy_connector")]
-    let should_bypass_proxy = url
-        .as_str()
-        .starts_with(&state.conf.connectors.dummyconnector.base_url)
-        || proxy_bypass_urls(
-            state.conf.key_manager.get_inner(),
-            &state.conf.locker,
-            &state.conf.proxy.bypass_proxy_urls,
-        )
-        .contains(&url.to_string());
-    #[cfg(not(feature = "dummy_connector"))]
-    let should_bypass_proxy = proxy_bypass_urls(
-        &state.conf.key_manager.get_inner(),
-        &state.conf.locker,
-        &state.conf.proxy.bypass_proxy_urls,
-    )
-    .contains(&url.to_string());
     let client = client::create_client(
         &state.conf.proxy,
-        should_bypass_proxy,
         request.certificate,
         request.certificate_key,
     )?;

crates/router/src/services/api/client.rs

@@ -10,18 +10,16 @@ use router_env::tracing_actix_web::RequestId;
 
 use super::{request::Maskable, Request};
 use crate::{
-    configs::settings::{Locker, Proxy},
+    configs::settings::Proxy,
-    consts::{BASE64_ENGINE, LOCKER_HEALTH_CALL_PATH},
+    consts::BASE64_ENGINE,
     core::errors::{ApiClientError, CustomResult},
-    routes::{app::settings::KeyManagerConfig, SessionState},
+    routes::SessionState,
 };
 
-static NON_PROXIED_CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
+static DEFAULT_CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
-static PROXIED_CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
 
 fn get_client_builder(
     proxy_config: &Proxy,
-    should_bypass_proxy: bool,
 ) -> CustomResult<reqwest::ClientBuilder, ApiClientError> {
     let mut client_builder = reqwest::Client::builder()
         .redirect(reqwest::redirect::Policy::none())
@@ -31,16 +29,16 @@ fn get_client_builder(
                 .unwrap_or_default(),
         ));
 
-    if should_bypass_proxy {
+    let proxy_exclusion_config =
-        return Ok(client_builder);
+        reqwest::NoProxy::from_string(&proxy_config.bypass_proxy_hosts.clone().unwrap_or_default());
-    }
 
     // Proxy all HTTPS traffic through the configured HTTPS proxy
     if let Some(url) = proxy_config.https_url.as_ref() {
         client_builder = client_builder.proxy(
             reqwest::Proxy::https(url)
                 .change_context(ApiClientError::InvalidProxyConfiguration)
-                .attach_printable("HTTPS proxy configuration error")?,
+                .attach_printable("HTTPS proxy configuration error")?
+                .no_proxy(proxy_exclusion_config.clone()),
         );
     }
 
@@ -49,44 +47,35 @@ fn get_client_builder(
         client_builder = client_builder.proxy(
             reqwest::Proxy::http(url)
                 .change_context(ApiClientError::InvalidProxyConfiguration)
-                .attach_printable("HTTP proxy configuration error")?,
+                .attach_printable("HTTP proxy configuration error")?
+                .no_proxy(proxy_exclusion_config),
         );
     }
 
     Ok(client_builder)
 }
 
-fn get_base_client(
+fn get_base_client(proxy_config: &Proxy) -> CustomResult<reqwest::Client, ApiClientError> {
-    proxy_config: &Proxy,
+    Ok(DEFAULT_CLIENT
-    should_bypass_proxy: bool,
+        .get_or_try_init(|| {
-) -> CustomResult<reqwest::Client, ApiClientError> {
+            get_client_builder(proxy_config)?
-    Ok(if should_bypass_proxy
+                .build()
-        || (proxy_config.http_url.is_none() && proxy_config.https_url.is_none())
+                .change_context(ApiClientError::ClientConstructionFailed)
-    {
+                .attach_printable("Failed to construct base client")
-        &NON_PROXIED_CLIENT
+        })?
-    } else {
+        .clone())
-        &PROXIED_CLIENT
-    }
-    .get_or_try_init(|| {
-        get_client_builder(proxy_config, should_bypass_proxy)?
-            .build()
-            .change_context(ApiClientError::ClientConstructionFailed)
-            .attach_printable("Failed to construct base client")
-    })?
-    .clone())
 }
 
 // We may need to use outbound proxy to connect to external world.
 // Precedence will be the environment variables, followed by the config.
 pub fn create_client(
     proxy_config: &Proxy,
-    should_bypass_proxy: bool,
     client_certificate: Option<masking::Secret<String>>,
     client_certificate_key: Option<masking::Secret<String>>,
 ) -> CustomResult<reqwest::Client, ApiClientError> {
     match (client_certificate, client_certificate_key) {
         (Some(encoded_certificate), Some(encoded_certificate_key)) => {
-            let client_builder = get_client_builder(proxy_config, should_bypass_proxy)?;
+            let client_builder = get_client_builder(proxy_config)?;
 
             let identity = create_identity_from_certificate_and_key(
                 encoded_certificate.clone(),
@@ -105,7 +94,7 @@ pub fn create_client(
                 .change_context(ApiClientError::ClientConstructionFailed)
                 .attach_printable("Failed to construct client with certificate and certificate key")
         }
-        _ => get_base_client(proxy_config, should_bypass_proxy),
+        _ => get_base_client(proxy_config),
     }
 }
 
@@ -145,35 +134,6 @@ pub fn create_certificate(
         .change_context(ApiClientError::CertificateDecodeFailed)
 }
 
-pub fn proxy_bypass_urls(
-    key_manager: &KeyManagerConfig,
-    locker: &Locker,
-    config_whitelist: &[String],
-) -> Vec<String> {
-    let key_manager_host = key_manager.url.to_owned();
-    let locker_host = locker.host.to_owned();
-    let locker_host_rs = locker.host_rs.to_owned();
-
-    let proxy_list = [
-        format!("{locker_host}/cards/add"),
-        format!("{locker_host}/cards/fingerprint"),
-        format!("{locker_host}/cards/retrieve"),
-        format!("{locker_host}/cards/delete"),
-        format!("{locker_host_rs}/cards/add"),
-        format!("{locker_host_rs}/cards/retrieve"),
-        format!("{locker_host_rs}/cards/delete"),
-        format!("{locker_host_rs}{}", LOCKER_HEALTH_CALL_PATH),
-        format!("{locker_host}/card/addCard"),
-        format!("{locker_host}/card/getCard"),
-        format!("{locker_host}/card/deleteCard"),
-        format!("{key_manager_host}/data/encrypt"),
-        format!("{key_manager_host}/data/decrypt"),
-        format!("{key_manager_host}/key/create"),
-        format!("{key_manager_host}/key/rotate"),
-    ];
-    [&proxy_list, config_whitelist].concat().to_vec()
-}
-
 pub trait RequestBuilder: Send + Sync {
     fn json(&mut self, body: serde_json::Value);
     fn url_encoded_form(&mut self, body: serde_json::Value);
@@ -201,6 +161,7 @@ where
         method: Method,
         url: String,
     ) -> CustomResult<Box<dyn RequestBuilder>, ApiClientError>;
+
     fn request_with_certificate(
         &self,
         method: Method,
@@ -218,7 +179,9 @@ where
     ) -> CustomResult<reqwest::Response, ApiClientError>;
 
     fn add_request_id(&mut self, request_id: RequestId);
+
     fn get_request_id(&self) -> Option<String>;
+
     fn add_flow_name(&mut self, flow_name: String);
 }
 
@@ -226,60 +189,31 @@ dyn_clone::clone_trait_object!(ApiClient);
 
 #[derive(Clone)]
 pub struct ProxyClient {
-    proxy_client: reqwest::Client,
+    proxy_config: Proxy,
-    non_proxy_client: reqwest::Client,
+    client: reqwest::Client,
-    whitelisted_urls: Vec<String>,
     request_id: Option<String>,
 }
 
 impl ProxyClient {
-    pub fn new(
+    pub fn new(proxy_config: &Proxy) -> CustomResult<Self, ApiClientError> {
-        proxy_config: Proxy,
+        let client = get_client_builder(proxy_config)?
-        whitelisted_urls: Vec<String>,
-    ) -> CustomResult<Self, ApiClientError> {
-        let non_proxy_client = reqwest::Client::builder()
-            .redirect(reqwest::redirect::Policy::none())
-            .build()
-            .change_context(ApiClientError::ClientConstructionFailed)?;
-
-        let mut proxy_builder =
-            reqwest::Client::builder().redirect(reqwest::redirect::Policy::none());
-
-        if let Some(url) = proxy_config.https_url.as_ref() {
-            proxy_builder = proxy_builder.proxy(
-                reqwest::Proxy::https(url)
-                    .change_context(ApiClientError::InvalidProxyConfiguration)?,
-            );
-        }
-
-        if let Some(url) = proxy_config.http_url.as_ref() {
-            proxy_builder = proxy_builder.proxy(
-                reqwest::Proxy::http(url)
-                    .change_context(ApiClientError::InvalidProxyConfiguration)?,
-            );
-        }
-
-        let proxy_client = proxy_builder
             .build()
             .change_context(ApiClientError::InvalidProxyConfiguration)?;
         Ok(Self {
-            proxy_client,
+            proxy_config: proxy_config.clone(),
-            non_proxy_client,
+            client,
-            whitelisted_urls,
             request_id: None,
         })
     }
 
     pub fn get_reqwest_client(
         &self,
-        base_url: String,
         client_certificate: Option<masking::Secret<String>>,
         client_certificate_key: Option<masking::Secret<String>>,
     ) -> CustomResult<reqwest::Client, ApiClientError> {
         match (client_certificate, client_certificate_key) {
             (Some(certificate), Some(certificate_key)) => {
-                let client_builder =
+                let client_builder = get_client_builder(&self.proxy_config)?;
-                    reqwest::Client::builder().redirect(reqwest::redirect::Policy::none());
                 let identity =
                     create_identity_from_certificate_and_key(certificate, certificate_key)?;
                 Ok(client_builder
@@ -290,13 +224,7 @@ impl ProxyClient {
                         "Failed to construct client with certificate and certificate key",
                     )?)
             }
-            (_, _) => {
+            (_, _) => Ok(self.client.clone()),
-                if self.whitelisted_urls.contains(&base_url) {
-                    Ok(self.non_proxy_client.clone())
-                } else {
-                    Ok(self.proxy_client.clone())
-                }
-            }
         }
     }
 }
@@ -355,8 +283,6 @@ impl RequestBuilder for RouterRequestBuilder {
     }
 }
 
-// TODO: remove this when integrating this trait
-#[allow(dead_code)]
 #[async_trait::async_trait]
 impl ApiClient for ProxyClient {
     fn request(
@@ -375,7 +301,7 @@ impl ApiClient for ProxyClient {
         certificate_key: Option<masking::Secret<String>>,
     ) -> CustomResult<Box<dyn RequestBuilder>, ApiClientError> {
         let client_builder = self
-            .get_reqwest_client(url.clone(), certificate, certificate_key)
+            .get_reqwest_client(certificate, certificate_key)
             .change_context(ApiClientError::ClientConstructionFailed)?;
         Ok(Box::new(RouterRequestBuilder {
             inner: Some(client_builder.request(method, url)),

crates/router/src/services/openidconnect.rs

@@ -155,7 +155,7 @@ async fn get_oidc_reqwest_client(
     state: &SessionState,
     request: oidc::HttpRequest,
 ) -> Result<oidc::HttpResponse, ApiClientError> {
-    let client = client::create_client(&state.conf.proxy, false, None, None)
+    let client = client::create_client(&state.conf.proxy, None, None)
         .map_err(|e| e.current_context().to_owned())?;
 
     let mut request_builder = client