mirror of
https://github.com/labmlai/annotated_deep_learning_paper_implementations.git
synced 2025-11-01 03:43:09 +08:00
14 lines
277 B
Python
14 lines
277 B
Python
import torch
|
|
from torch import nn
|
|
|
|
from labml_helpers.module import Module
|
|
|
|
|
|
class Swish(Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.sigmoid = nn.Sigmoid()
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
return x * self.sigmoid(x)
|