From d747e463a4097e41727678c558ae342f6ac5132d Mon Sep 17 00:00:00 2001 From: Varuna Jayasiri Date: Wed, 23 Feb 2022 15:10:49 +0530 Subject: [PATCH] rope links --- labml_nn/__init__.py | 1 + labml_nn/transformers/__init__.py | 3 +++ readme.md | 1 + 3 files changed, 5 insertions(+) diff --git a/labml_nn/__init__.py b/labml_nn/__init__.py index 1280e454..cd1e9118 100644 --- a/labml_nn/__init__.py +++ b/labml_nn/__init__.py @@ -22,6 +22,7 @@ implementations. * [Transformer building blocks](transformers/models.html) * [Transformer XL](transformers/xl/index.html) * [Relative multi-headed attention](transformers/xl/relative_mha.html) +* [Rotary Positional Embeddings](transformers/rope/index.html) * [Compressive Transformer](transformers/compressive/index.html) * [GPT Architecture](transformers/gpt/index.html) * [GLU Variants](transformers/glu_variants/simple.html) diff --git a/labml_nn/transformers/__init__.py b/labml_nn/transformers/__init__.py index 9a7c5242..7d836880 100644 --- a/labml_nn/transformers/__init__.py +++ b/labml_nn/transformers/__init__.py @@ -22,6 +22,9 @@ and derivatives and enhancements of it. This implements Transformer XL model using [relative multi-head attention](xl/relative_mha.html) +## [Rotary Positional Embeddings](rope/index.html) +This implements Rotary Positional Embeddings (RoPE) + ## [Compressive Transformer](compressive/index.html) This is an implementation of compressive transformer diff --git a/readme.md b/readme.md index 176b1b0e..d1c860fc 100644 --- a/readme.md +++ b/readme.md @@ -24,6 +24,7 @@ implementations almost weekly. * [Transformer building blocks](https://nn.labml.ai/transformers/models.html) * [Transformer XL](https://nn.labml.ai/transformers/xl/index.html) * [Relative multi-headed attention](https://nn.labml.ai/transformers/xl/relative_mha.html) +* [Rotary Positional Embeddings](https://nn.labml.ai/transformers/rope/index.html) * [Compressive Transformer](https://nn.labml.ai/transformers/compressive/index.html) * [GPT Architecture](https://nn.labml.ai/transformers/gpt/index.html) * [GLU Variants](https://nn.labml.ai/transformers/glu_variants/simple.html)