learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-11-04 06:16:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

math comment fix

Browse Source
This commit is contained in:
Varuna Jayasiri
2021-10-21 13:25:11 +05:30
parent 77bf55e03a
commit 14fb057045
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
labml_nn/transformers/fast_weights/__init__.py
View File

@ -181,7 +181,7 @@ class FastWeightsAttention(Module):
The model first retrieves the current value The model first retrieves the current value
$\bar{v}^{(i)}$ paired with the key $k^{(i)}$. $\bar{v}^{(i)}$ paired with the key $k^{(i)}$.
Then stores a combination $v^{(i)}_{new}$ Then stores a combination $v^{(i)}_{new}$
of the retrieved value $\bar{v}^{̄(i)}$ and the input $v^{(i)}$. of the retrieved value $\bar{v}^{(i)}$ and the input $v^{(i)}$.
\begin{align} \begin{align}
k^{(i)}, v^{(i)}, q^{(i)} &= k^{(i)}, v^{(i)}, q^{(i)} &=

2
labml_nn/transformers/positional_encoding.py
View File

@ -50,7 +50,7 @@ def get_positional_encoding(d_model: int, max_len: int = 5000):
position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1) position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)
# $2 * i$ # $2 * i$
two_i = torch.arange(0, d_model, 2, dtype=torch.float32) two_i = torch.arange(0, d_model, 2, dtype=torch.float32)
# $10000^{\frac{2i}{d_{model}}$ # $10000^{\frac{2i}{d_{model}}}$
div_term = torch.exp(two_i * -(math.log(10000.0) / d_model)) div_term = torch.exp(two_i * -(math.log(10000.0) / d_model))
# $PE_{p,2i} = sin\Bigg(\frac{p}{10000^{\frac{2i}{d_{model}}}}\Bigg)$ # $PE_{p,2i} = sin\Bigg(\frac{p}{10000^{\frac{2i}{d_{model}}}}\Bigg)$
encodings[:, 0::2] = torch.sin(position * div_term) encodings[:, 0::2] = torch.sin(position * div_term)