feat(proxy): add support to pass proxy bypass urls from configs (#5322)

config/config.example.toml

@@ -26,6 +26,7 @@ certificate = "/path/to/certificate.pem"
 # http_url = "http proxy url"   # Proxy all HTTP traffic via this proxy
 # https_url = "https proxy url" # Proxy all HTTPS traffic via this proxy
 idle_pool_connection_timeout = 90 # Timeout for idle pool connections (defaults to 90s)
+bypass_proxy_urls = []            # A list of URLs that should bypass the proxy
 
 
 # Configuration for the Key Manager Service
@@ -284,30 +285,30 @@ stripe = { banks = "alior_bank,bank_millennium,bank_nowy_bfg_sa,bank_pekao_sa,ba
 wallets = ["klarna", "mifinity", "braintree", "applepay"]
 rewards = ["cashtocode", "zen"]
 cards = [
-    "adyen",
-    "adyenplatform",
-    "authorizedotnet",
-    "coinbase",
-    "cryptopay",
-    "braintree",
-    "checkout",
-    "cybersource",
-    "datatrans",
-    "globalpay",
-    "globepay",
-    "gocardless",
-    "gpayments",
-    "helcim",
-    "mollie",
-    "paypal",
-    "shift4",
-    "square",
-    "stax",
-    "stripe",
-    "threedsecureio",
-    "worldpay",
-    "zen",
-    "zsl",
+  "adyen",
+  "adyenplatform",
+  "authorizedotnet",
+  "coinbase",
+  "cryptopay",
+  "braintree",
+  "checkout",
+  "cybersource",
+  "datatrans",
+  "globalpay",
+  "globepay",
+  "gocardless",
+  "gpayments",
+  "helcim",
+  "mollie",
+  "paypal",
+  "shift4",
+  "square",
+  "stax",
+  "stripe",
+  "threedsecureio",
+  "worldpay",
+  "zen",
+  "zsl",
 ]
 
 # Scheduler settings provides a point to modify the behaviour of scheduler flow.
@@ -438,13 +439,13 @@ card.debit = { connector_list = "cybersource" }  # Update Mandate supported paym
 # Required fields info used while listing the payment_method_data
 [required_fields.pay_later] # payment_method = "pay_later"
 afterpay_clearpay = { fields = { stripe = [ # payment_method_type = afterpay_clearpay, connector = "stripe"
-    # Required fields vector with its respective display name in front-end and field_type
-    { required_field = "shipping.address.first_name", display_name = "first_name", field_type = "text" },
-    { required_field = "shipping.address.last_name", display_name = "last_name", field_type = "text" },
-    { required_field = "shipping.address.country", display_name = "country", field_type = { drop_down = { options = [
-        "US",
-        "IN",
-    ] } } },
+  # Required fields vector with its respective display name in front-end and field_type
+  { required_field = "shipping.address.first_name", display_name = "first_name", field_type = "text" },
+  { required_field = "shipping.address.last_name", display_name = "last_name", field_type = "text" },
+  { required_field = "shipping.address.country", display_name = "country", field_type = { drop_down = { options = [
+    "US",
+    "IN",
+  ] } } },
 ] } }
 
 [payouts]
@@ -484,7 +485,7 @@ boleto = { country = "BR", currency = "BRL" }
 open_banking_uk = { country = "DE,GB,AT,BE,CY,EE,ES,FI,FR,GR,HR,IE,IT,LT,LU,LV,MT,NL,PT,SI,SK,BG,CZ,DK,HU,NO,PL,RO,SE,AU,BR", currency = "EUR,GBP,DKK,NOK,PLN,SEK,AUD,BRL" }
 
 [pm_filters.razorpay]
-upi_collect = {country = "IN", currency = "INR"}
+upi_collect = { country = "IN", currency = "INR" }
 
 [pm_filters.zen]
 credit = { not_available_flows = { capture_method = "manual" } }
@@ -586,7 +587,7 @@ card = "credit,debit"
 
 #Payout Method Filters Based on Country and Currency
 [payout_method_filters.adyenplatform]
-sepa = { country = "ES,SK,AT,NL,DE,BE,FR,FI,PT,IE,EE,LT,LV,IT,CZ,DE,HU,NO,PL,SE,GB,CH" , currency = "EUR,CZK,DKK,HUF,NOK,PLN,SEK,GBP,CHF" }
+sepa = { country = "ES,SK,AT,NL,DE,BE,FR,FI,PT,IE,EE,LT,LV,IT,CZ,DE,HU,NO,PL,SE,GB,CH", currency = "EUR,CZK,DKK,HUF,NOK,PLN,SEK,GBP,CHF" }
 
 [payout_method_filters.stripe]
 ach = { country = "US", currency = "USD" }
@@ -636,19 +637,19 @@ enabled = true                      # Switch to enable or disable PayPal onboard
 source = "logs" # The event sink to push events supports kafka or logs (stdout)
 
 [events.kafka]
-brokers = []                               # Kafka broker urls for bootstrapping the client
-fraud_check_analytics_topic = "topic"      # Kafka topic to be used for FraudCheck events
-intent_analytics_topic = "topic"           # Kafka topic to be used for PaymentIntent events
-attempt_analytics_topic = "topic"          # Kafka topic to be used for PaymentAttempt events
-refund_analytics_topic = "topic"           # Kafka topic to be used for Refund events
-api_logs_topic = "topic"                   # Kafka topic to be used for incoming api events
-connector_logs_topic = "topic"             # Kafka topic to be used for connector api events
-outgoing_webhook_logs_topic = "topic"      # Kafka topic to be used for outgoing webhook events
-dispute_analytics_topic = "topic"          # Kafka topic to be used for Dispute events
-audit_events_topic = "topic"               # Kafka topic to be used for Payment Audit events
-payout_analytics_topic = "topic"           # Kafka topic to be used for Payouts and PayoutAttempt events
-consolidated_events_topic = "topic"        # Kafka topic to be used for Consolidated events
-authentication_analytics_topic = "topic"   # Kafka topic to be used for Authentication events
+brokers = []                             # Kafka broker urls for bootstrapping the client
+fraud_check_analytics_topic = "topic"    # Kafka topic to be used for FraudCheck events
+intent_analytics_topic = "topic"         # Kafka topic to be used for PaymentIntent events
+attempt_analytics_topic = "topic"        # Kafka topic to be used for PaymentAttempt events
+refund_analytics_topic = "topic"         # Kafka topic to be used for Refund events
+api_logs_topic = "topic"                 # Kafka topic to be used for incoming api events
+connector_logs_topic = "topic"           # Kafka topic to be used for connector api events
+outgoing_webhook_logs_topic = "topic"    # Kafka topic to be used for outgoing webhook events
+dispute_analytics_topic = "topic"        # Kafka topic to be used for Dispute events
+audit_events_topic = "topic"             # Kafka topic to be used for Payment Audit events
+payout_analytics_topic = "topic"         # Kafka topic to be used for Payouts and PayoutAttempt events
+consolidated_events_topic = "topic"      # Kafka topic to be used for Consolidated events
+authentication_analytics_topic = "topic" # Kafka topic to be used for Authentication events
 
 # File storage configuration
 [file_storage]

config/deployments/env_specific.toml

@@ -43,9 +43,9 @@ partner_id = "paypal_partner_id"
 
 [connector_request_reference_id_config]
 merchant_ids_send_payment_id_as_connector_request_id = [
-    "merchant_id_1",
-    "merchant_id_2",
-    "etc.,",
+  "merchant_id_1",
+  "merchant_id_2",
+  "etc.,",
 ]
 
 [cors]
@@ -70,19 +70,19 @@ sts_role_session_name = "" # An identifier for the assumed role session, used to
 source = "logs" # The event sink to push events supports kafka or logs (stdout)
 
 [events.kafka]
-brokers = []                               # Kafka broker urls for bootstrapping the client
-intent_analytics_topic = "topic"           # Kafka topic to be used for PaymentIntent events
-attempt_analytics_topic = "topic"          # Kafka topic to be used for PaymentAttempt events
-refund_analytics_topic = "topic"           # Kafka topic to be used for Refund events
-api_logs_topic = "topic"                   # Kafka topic to be used for incoming api events
-connector_logs_topic = "topic"             # Kafka topic to be used for connector api events
-outgoing_webhook_logs_topic = "topic"      # Kafka topic to be used for outgoing webhook events
-dispute_analytics_topic = "topic"          # Kafka topic to be used for Dispute events
-audit_events_topic = "topic"               # Kafka topic to be used for Payment Audit events
-payout_analytics_topic = "topic"           # Kafka topic to be used for Payouts and PayoutAttempt events
-consolidated_events_topic = "topic"        # Kafka topic to be used for Consolidated events
-authentication_analytics_topic = "topic"   # Kafka topic to be used for Authentication events
-fraud_check_analytics_topic = "topic"      # Kafka topic to be used for Fraud Check events
+brokers = []                             # Kafka broker urls for bootstrapping the client
+intent_analytics_topic = "topic"         # Kafka topic to be used for PaymentIntent events
+attempt_analytics_topic = "topic"        # Kafka topic to be used for PaymentAttempt events
+refund_analytics_topic = "topic"         # Kafka topic to be used for Refund events
+api_logs_topic = "topic"                 # Kafka topic to be used for incoming api events
+connector_logs_topic = "topic"           # Kafka topic to be used for connector api events
+outgoing_webhook_logs_topic = "topic"    # Kafka topic to be used for outgoing webhook events
+dispute_analytics_topic = "topic"        # Kafka topic to be used for Dispute events
+audit_events_topic = "topic"             # Kafka topic to be used for Payment Audit events
+payout_analytics_topic = "topic"         # Kafka topic to be used for Payouts and PayoutAttempt events
+consolidated_events_topic = "topic"      # Kafka topic to be used for Consolidated events
+authentication_analytics_topic = "topic" # Kafka topic to be used for Authentication events
+fraud_check_analytics_topic = "topic"    # Kafka topic to be used for Fraud Check events
 
 # File storage configuration
 [file_storage]
@@ -190,6 +190,7 @@ redis_expiry = 900          # Redis expiry time in milliseconds
 [proxy]
 http_url = "http://proxy_http_url"    # Outgoing proxy http URL to proxy the HTTP traffic
 https_url = "https://proxy_https_url" # Outgoing proxy https URL to proxy the HTTPS traffic
+bypass_proxy_urls = []                # A list of URLs that should bypass the proxy
 
 # Redis credentials
 [redis]
@@ -210,8 +211,8 @@ unresponsive_timeout = 10 # An optional timeout for Unresponsive commands in sec
 max_feed_count = 200 # The maximum number of frames that will be fed to a socket before flushing.
 cluster_enabled = true # boolean
 cluster_urls = [
-    "redis.cluster.uri-1:8080",
-    "redis.cluster.uri-2:4115",
+  "redis.cluster.uri-1:8080",
+  "redis.cluster.uri-2:4115",
 ] # List of redis cluster urls
 
 # Replica SQL data store credentials
@@ -295,7 +296,7 @@ enabled = false
 global_tenant = { schema = "public", redis_key_prefix = "" }
 
 [multitenancy.tenants]
-public = { name = "hyperswitch", base_url = "http://localhost:8080", schema = "public", redis_key_prefix = "", clickhouse_database = "default"}
+public = { name = "hyperswitch", base_url = "http://localhost:8080", schema = "public", redis_key_prefix = "", clickhouse_database = "default" }
 
 [user_auth_methods]
 encryption_key = "user_auth_table_encryption_key" # Encryption key used for encrypting data in user_authentication_methods table

crates/router/src/bin/scheduler.rs

@@ -38,7 +38,7 @@ async fn main() -> CustomResult<(), ProcessTrackerError> {
     let api_client = Box::new(
         services::ProxyClient::new(
             conf.proxy.clone(),
-            services::proxy_bypass_urls(&conf.locker),
+            services::proxy_bypass_urls(&conf.locker, &conf.proxy.bypass_proxy_urls),
         )
         .change_context(ProcessTrackerError::ConfigurationError)?,
     );

crates/router/src/configs/defaults.rs

@@ -55,6 +55,7 @@ impl Default for super::settings::Proxy {
             http_url: Default::default(),
             https_url: Default::default(),
             idle_pool_connection_timeout: Some(90),
+            bypass_proxy_urls: Vec::new(),
         }
     }
 }

crates/router/src/configs/settings.rs

@@ -582,6 +582,7 @@ pub struct Proxy {
     pub http_url: Option<String>,
     pub https_url: Option<String>,
     pub idle_pool_connection_timeout: Option<u64>,
+    pub bypass_proxy_urls: Vec<String>,
 }
 
 #[derive(Debug, Deserialize, Clone)]
@@ -734,6 +735,7 @@ impl Settings<SecuredSecret> {
                     .with_list_parse_key("log.telemetry.route_to_trace")
                     .with_list_parse_key("redis.cluster_urls")
                     .with_list_parse_key("events.kafka.brokers")
+                    .with_list_parse_key("proxy.bypass_proxy_urls")
                     .with_list_parse_key("connectors.supported.wallets")
                     .with_list_parse_key("connector_request_reference_id_config.merchant_ids_send_payment_id_as_connector_request_id"),
 

crates/router/src/lib.rs

@@ -192,7 +192,7 @@ pub async fn start_server(conf: settings::Settings<SecuredSecret>) -> Applicatio
     let api_client = Box::new(
         services::ProxyClient::new(
             conf.proxy.clone(),
-            services::proxy_bypass_urls(&conf.locker),
+            services::proxy_bypass_urls(&conf.locker, &conf.proxy.bypass_proxy_urls),
         )
         .map_err(|error| {
             errors::ApplicationError::ApiClientError(error.current_context().clone())

crates/router/src/services/api.rs

@@ -473,9 +473,12 @@ pub async fn send_request(
     let should_bypass_proxy = url
         .as_str()
         .starts_with(&state.conf.connectors.dummyconnector.base_url)
-        || proxy_bypass_urls(&state.conf.locker).contains(&url.to_string());
+        || proxy_bypass_urls(&state.conf.locker, &state.conf.proxy.bypass_proxy_urls)
+            .contains(&url.to_string());
     #[cfg(not(feature = "dummy_connector"))]
-    let should_bypass_proxy = proxy_bypass_urls(&state.conf.locker).contains(&url.to_string());
+    let should_bypass_proxy =
+        proxy_bypass_urls(&state.conf.locker, &state.conf.proxy.bypass_proxy_urls)
+            .contains(&url.to_string());
     let client = client::create_client(
         &state.conf.proxy,
         should_bypass_proxy,

crates/router/src/services/api/client.rs

@@ -110,10 +110,11 @@ pub fn create_client(
     }
 }
 
-pub fn proxy_bypass_urls(locker: &Locker) -> Vec<String> {
+pub fn proxy_bypass_urls(locker: &Locker, config_whitelist: &[String]) -> Vec<String> {
     let locker_host = locker.host.to_owned();
     let locker_host_rs = locker.host_rs.to_owned();
-    vec![
+
+    let proxy_list = [
         format!("{locker_host}/cards/add"),
         format!("{locker_host}/cards/fingerprint"),
         format!("{locker_host}/cards/retrieve"),
@@ -125,7 +126,9 @@ pub fn proxy_bypass_urls(locker: &Locker) -> Vec<String> {
         format!("{locker_host}/card/addCard"),
         format!("{locker_host}/card/getCard"),
         format!("{locker_host}/card/deleteCard"),
-    ]
+    ];
+
+    [&proxy_list, config_whitelist].concat().to_vec()
 }
 
 pub trait RequestBuilder: Send + Sync {