feat: add unresponsive timeout for fred (#3369)

crates/redis_interface/src/lib.rs

@@ -132,6 +132,11 @@ impl RedisConnectionPool {
             },
         };
 
+        let connection_config = fred::types::ConnectionConfig {
+            unresponsive_timeout: std::time::Duration::from_secs(conf.unresponsive_timeout),
+            ..fred::types::ConnectionConfig::default()
+        };
+
         if !conf.use_legacy_version {
             config.version = fred::types::RespVersion::RESP3;
         }
@@ -151,7 +156,7 @@ impl RedisConnectionPool {
         let pool = fred::prelude::RedisPool::new(
             config,
             Some(perf),
-            None,
+            Some(connection_config),
             Some(reconnect_policy),
             conf.pool_size,
         )
@@ -201,6 +206,15 @@ impl RedisConnectionPool {
             }
         }
     }
+
+    pub async fn on_unresponsive(&self) {
+        let _ = self.pool.clients().iter().map(|client| {
+            client.on_unresponsive(|server| {
+                logger::warn!(redis_server =?server.host, "Redis server is unresponsive");
+                Ok(())
+            })
+        });
+    }
 }
 
 struct RedisConfig {

crates/redis_interface/src/types.rs

@@ -57,6 +57,7 @@ pub struct RedisSettings {
     pub max_in_flight_commands: u64,
     pub default_command_timeout: u64,
     pub max_feed_count: u64,
+    pub unresponsive_timeout: u64,
 }
 
 impl RedisSettings {
@@ -76,7 +77,17 @@ impl RedisSettings {
                 "Redis `cluster_urls` must be specified if `cluster_enabled` is `true`".into(),
             ))
             .into_report()
-        })
+        })?;
+
+        when(
+            self.default_command_timeout < self.unresponsive_timeout,
+            || {
+                Err(errors::RedisError::InvalidConfiguration(
+                    "Unresponsive timeout cannot be greater than the command timeout".into(),
+                ))
+                .into_report()
+            },
+        )
     }
 }
 
@@ -97,8 +108,9 @@ impl Default for RedisSettings {
             auto_pipeline: true,
             disable_auto_backpressure: false,
             max_in_flight_commands: 5000,
-            default_command_timeout: 0,
+            default_command_timeout: 30,
             max_feed_count: 200,
+            unresponsive_timeout: 10,
         }
     }
 }