fix: throw 500 error when redis goes down (#531)

crates/router/src/services.rs

@@ -4,12 +4,17 @@ pub mod authentication;
 pub mod encryption;
 pub mod logger;
 
-use std::sync::Arc;
+use std::sync::{atomic, Arc};
+
+use redis_interface::errors::RedisError;
 
 pub use self::api::*;
 #[cfg(feature = "basilisk")]
 pub use self::encryption::*;
-use crate::connection::{diesel_make_pg_pool, PgPool};
+use crate::{
+    connection::{diesel_make_pg_pool, PgPool},
+    core::errors,
+};
 
 #[derive(Clone)]
 pub struct Store {
@@ -30,11 +35,18 @@ pub(crate) struct StoreConfig {
 
 impl Store {
     pub async fn new(config: &crate::configs::settings::Settings, test_transaction: bool) -> Self {
+        let redis_conn = Arc::new(crate::connection::redis_connection(config).await);
+        let redis_clone = redis_conn.clone();
+
+        tokio::spawn(async move {
+            redis_clone.on_error().await;
+        });
+
         Self {
             master_pool: diesel_make_pg_pool(&config.master_database, test_transaction).await,
             #[cfg(feature = "olap")]
             replica_pool: diesel_make_pg_pool(&config.replica_database, test_transaction).await,
-            redis_conn: Arc::new(crate::connection::redis_connection(config).await),
+            redis_conn,
             #[cfg(feature = "kv_store")]
             config: StoreConfig {
                 drainer_stream_name: config.drainer.stream_name.clone(),
@@ -49,6 +61,20 @@ impl Store {
         format!("{{{}}}_{}", shard_key, self.config.drainer_stream_name,)
     }
 
+    pub fn redis_conn(
+        &self,
+    ) -> errors::CustomResult<Arc<redis_interface::RedisConnectionPool>, RedisError> {
+        if self
+            .redis_conn
+            .is_redis_available
+            .load(atomic::Ordering::SeqCst)
+        {
+            Ok(self.redis_conn.clone())
+        } else {
+            Err(RedisError::RedisConnectionError.into())
+        }
+    }
+
     #[cfg(feature = "kv_store")]
     pub(crate) async fn push_to_drainer_stream<T>(
         &self,