learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-08-26 08:41:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

gpt notes

Browse Source
This commit is contained in:
Varuna Jayasiri
2021-01-14 09:37:12 +05:30
parent 4c769128cb
commit 2927aa217b
6 changed files with 215 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
labml_nn/optimizers/ada_belief.py
View File

@ -51,7 +51,7 @@ class AdaBelief(RAdam):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-16,
weight_decay: WeightDecay = WeightDecay(), amsgrad=False,
degenerated_to_sgd=True,
degenerate_to_sgd=True,
rectify=True, defaults=None):
"""
### Initialize the optimizer
@ -71,7 +71,7 @@ class AdaBelief(RAdam):
"""
defaults = {} if defaults is None else defaults
super().__init__(params, lr, betas, eps, weight_decay, amsgrad, degenerated_to_sgd, defaults)
super().__init__(params, lr, betas, eps, weight_decay, amsgrad, degenerate_to_sgd, defaults)
self.rectify = rectify
def init_state(self, state: Dict[str, any], group: Dict[str, any], param: nn.Parameter):