learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-08-14 09:31:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

bias fix

Browse Source
This commit is contained in:
Varuna Jayasiri
2022-06-03 17:23:39 +05:30
parent 321339eeb0
commit 09d2210aac
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
labml_nn/transformers/rope/__init__.py
View File

@ -201,7 +201,7 @@ class RotaryPEMultiHeadAttention(MultiHeadAttention):
"""
def __init__(self, heads: int, d_model: int, rope_percentage: float = 0.5, dropout_prob: float = 0.0):
super().__init__(heads, d_model, dropout_prob, bias=False)
super().__init__(heads, d_model, dropout_prob)
# Rotary positional embedding layers
d_rope = int(self.d_k * rope_percentage)

2
labml_nn/transformers/rope/value_pe/__init__.py
View File

@ -174,7 +174,7 @@ class RotaryValuePEMultiHeadAttention(MultiHeadAttention):
def __init__(self, heads: int, d_model: int,
rope_percentage: float = 0.5, rope_value_percentage: float = 0.5,
dropout_prob: float = 0.0):
super().__init__(heads, d_model, dropout_prob, bias=False)
super().__init__(heads, d_model, dropout_prob)
# Rotary positional embedding layers
d_rope = int(self.d_k * rope_percentage)