learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-10-31 18:58:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

default warm up

Browse Source
This commit is contained in:
Varuna Jayasiri
2020-12-23 13:57:34 +05:30
parent 634809af3c
commit b05faa9b98
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
labml_nn/optimizers/configs.py
View File

@ -30,7 +30,7 @@ class OptimizerConfigs(BaseConfigs):
momentum: float = 0.5
amsgrad: bool = False
warmup: int = 0
warmup: int = 2_000
degenerated_to_sgd: bool = True
rectify: bool = True
d_model: int