diff --git a/docs/index.html b/docs/index.html index 1c500ed6..a6e0faf1 100644 --- a/docs/index.html +++ b/docs/index.html @@ -90,7 +90,8 @@
  • MLP-Mixer: An all-MLP Architecture for Vision
  • Pay Attention to MLPs (gMLP)
  • Vision Transformer (ViT)
    • -
  • Primer EZ
    • +
  • Primer EZ
    • +
  • Hourglass

    • ✨ Recurrent Highway Networks

    ✨ LSTM

    ✨ HyperNetworks - HyperLSTM

    diff --git a/docs/transformers/index.html b/docs/transformers/index.html index dce7dcc4..33989c35 100644 --- a/docs/transformers/index.html +++ b/docs/transformers/index.html @@ -104,13 +104,15 @@

    This is an implementation of the paper An Image Is Worth 16x16 Words: Transformers For Image Recognition At Scale.

    Primer EZ

    This is an implementation of the paper Primer: Searching for Efficient Transformers for Language Modeling.

    +

    Hourglass

    +

    This is an implementation of the paper Hierarchical Transformers Are More Efficient Language Models

    - 
    98from .configs import TransformerConfigs
-99from .models import TransformerLayer, Encoder, Decoder, Generator, EncoderDecoder
-100from .mha import MultiHeadAttention
-101from labml_nn.transformers.xl.relative_mha import RelativeMultiHeadAttention
    + 
    103from .configs import TransformerConfigs
+104from .models import TransformerLayer, Encoder, Decoder, Generator, EncoderDecoder
+105from .mha import MultiHeadAttention
+106from labml_nn.transformers.xl.relative_mha import RelativeMultiHeadAttention
    diff --git a/labml_nn/__init__.py b/labml_nn/__init__.py index 63a0413d..8e320220 100644 --- a/labml_nn/__init__.py +++ b/labml_nn/__init__.py @@ -36,6 +36,7 @@ implementations. * [Pay Attention to MLPs (gMLP)](transformers/gmlp/index.html) * [Vision Transformer (ViT)](transformers/vit/index.html) * [Primer EZ](transformers/primer_ez/index.html) +* [Hourglass](transformers/hourglass/index.html) #### ✨ [Recurrent Highway Networks](recurrent_highway_networks/index.html) diff --git a/labml_nn/transformers/__init__.py b/labml_nn/transformers/__init__.py index f5baad98..9a7c5242 100644 --- a/labml_nn/transformers/__init__.py +++ b/labml_nn/transformers/__init__.py @@ -93,6 +93,11 @@ This is an implementation of the paper This is an implementation of the paper [Primer: Searching for Efficient Transformers for Language Modeling](https://papers.labml.ai/paper/2109.08668). + +## [Hourglass](hour_glass/index.html) + +This is an implementation of the paper +[Hierarchical Transformers Are More Efficient Language Models](https://papers.labml.ai/paper/2110.13711) """ from .configs import TransformerConfigs diff --git a/readme.md b/readme.md index c454c36c..91a2376c 100644 --- a/readme.md +++ b/readme.md @@ -38,6 +38,7 @@ implementations almost weekly. * [Pay Attention to MLPs (gMLP)](https://nn.labml.ai/transformers/gmlp/index.html) * [Vision Transformer (ViT)](https://nn.labml.ai/transformers/vit/index.html) * [Primer EZ](https://nn.labml.ai/transformers/primer_ez/index.html) +* [Hourglass](https://nn.labml.ai/transformers/hourglass/index.html) #### ✨ [Recurrent Highway Networks](https://nn.labml.ai/recurrent_highway_networks/index.html)