mirror of
https://github.com/labmlai/annotated_deep_learning_paper_implementations.git
synced 2025-11-04 06:16:05 +08:00
19 lines
381 B
Python
19 lines
381 B
Python
"""
|
|
---
|
|
title: Utilities for Transformer
|
|
summary: A bunch of utility functions and classes for transformers.
|
|
---
|
|
|
|
# Utilities for Transformer
|
|
"""
|
|
|
|
import torch
|
|
|
|
|
|
def subsequent_mask(seq_len):
|
|
"""
|
|
## Subsequent mask to mask out data from future (subsequent) time steps
|
|
"""
|
|
mask = torch.tril(torch.ones(seq_len, seq_len)).to(torch.bool).unsqueeze(-1)
|
|
return mask
|