learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-10-31 18:58:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

🐛 positional encoding buffer

Browse Source
This commit is contained in:
Varuna Jayasiri
2020-09-27 16:40:22 +05:30
parent b2b305ff4d
commit 06e68d012c
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
labml_nn/transformers/positional_encoding.py
View File

@ -28,7 +28,7 @@ class PositionalEncoding(Module):
super().__init__() super().__init__()
self.dropout = nn.Dropout(dropout_prob) self.dropout = nn.Dropout(dropout_prob)
self.register_buffer('positional_encodings', get_positional_encoding(d_model, max_len)) self.register_buffer('positional_encodings', get_positional_encoding(d_model, max_len), False)
def __call__(self, x: torch.Tensor): def __call__(self, x: torch.Tensor):
pe = self.positional_encodings[:x.shape[0]].detach().requires_grad_(False) pe = self.positional_encodings[:x.shape[0]].detach().requires_grad_(False)