From b05faa9b98e7f58072c36c807e8b4ba8533c02c0 Mon Sep 17 00:00:00 2001
From: Varuna Jayasiri <vpjayasiri@gmail.com>
Date: Wed, 23 Dec 2020 13:57:34 +0530
Subject: [PATCH] default warm up

---
 labml_nn/optimizers/configs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/labml_nn/optimizers/configs.py b/labml_nn/optimizers/configs.py
index c8f0df75..ed1214fb 100644
--- a/labml_nn/optimizers/configs.py
+++ b/labml_nn/optimizers/configs.py
@@ -30,7 +30,7 @@ class OptimizerConfigs(BaseConfigs):
 
     momentum: float = 0.5
     amsgrad: bool = False
-    warmup: int = 0
+    warmup: int = 2_000
     degenerated_to_sgd: bool = True
     rectify: bool = True
     d_model: int