From 830161b299cfe0b653a5eea5175b4dc53a1da9f2 Mon Sep 17 00:00:00 2001
From: MrYXJ <529674242@qq.com>
Date: Tue, 14 Nov 2023 00:30:26 +0800
Subject: [PATCH] Update __init__.py

This formula is wrong, there is one symbol '-' missing in front of the 1, which will affect people's understanding when reading. What is expressed here is that the position of the ith token is increasing from -(i-1) to 0, so it should be -1.
---
 labml_nn/transformers/alibi/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/labml_nn/transformers/alibi/__init__.py b/labml_nn/transformers/alibi/__init__.py
index e8b9f4c7..9ab70f8f 100644
--- a/labml_nn/transformers/alibi/__init__.py
+++ b/labml_nn/transformers/alibi/__init__.py
@@ -19,7 +19,7 @@ Here's the attention formula for $i$-th token,
 
 \begin{align}
 \mathbf{a}_i
-&= \text{softmax} \bigg( \mathbf{q}_i \mathbf{K}^\top + m \cdot \big[-(i-1), \dots, 1, 0 \big] \bigg) \\
+&= \text{softmax} \bigg( \mathbf{q}_i \mathbf{K}^\top + m \cdot \big[-(i-1), \dots, -1, 0 \big] \bigg) \\
 &= \text{softmax} \bigg( \mathbf{q}_i \mathbf{K}^\top + m \cdot \big[0, 1, \dots, (i - 1) \big] \bigg)
 \end{align}