learning/annotated_deep_learning_paper_implementations
1
0
Fork 0
mirror of https://github.com/labmlai/annotated_deep_learning_paper_implementations.git synced 2025-08-16 10:51:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

bias fix

Browse Source
This commit is contained in:
Varuna Jayasiri
2022-06-03 17:23:39 +05:30
parent 321339eeb0
commit 09d2210aac
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
labml_nn/transformers/rope/__init__.py
View File

@ -201,7 +201,7 @@ class RotaryPEMultiHeadAttention(MultiHeadAttention):
""" """
def __init__(self, heads: int, d_model: int, rope_percentage: float = 0.5, dropout_prob: float = 0.0): def __init__(self, heads: int, d_model: int, rope_percentage: float = 0.5, dropout_prob: float = 0.0):
super().__init__(heads, d_model, dropout_prob, bias=False) super().__init__(heads, d_model, dropout_prob)
# Rotary positional embedding layers # Rotary positional embedding layers
d_rope = int(self.d_k * rope_percentage) d_rope = int(self.d_k * rope_percentage)

2
labml_nn/transformers/rope/value_pe/__init__.py
View File

@ -174,7 +174,7 @@ class RotaryValuePEMultiHeadAttention(MultiHeadAttention):
def __init__(self, heads: int, d_model: int, def __init__(self, heads: int, d_model: int,
rope_percentage: float = 0.5, rope_value_percentage: float = 0.5, rope_percentage: float = 0.5, rope_value_percentage: float = 0.5,
dropout_prob: float = 0.0): dropout_prob: float = 0.0):
super().__init__(heads, d_model, dropout_prob, bias=False) super().__init__(heads, d_model, dropout_prob)
# Rotary positional embedding layers # Rotary positional embedding layers
d_rope = int(self.d_k * rope_percentage) d_rope = int(self.d_k * rope_percentage)