mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Implement Tanh Gelu Approximation (#61439)
Summary:
1. Implements https://github.com/pytorch/pytorch/issues/39853
2. Adds approximate boolean flag to Gelu
3. Enables Tanh Gelu approximation
4. Adds double backward support for Gelu
5. Enable Tanh Gelu in NvFuser
```
def gelu(x, approximate : str = 'none'):
if approximate == 'tanh':
# sqrt(2/pi) = 0.7978845608028654
return 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * (x + 0.044715 * torch.pow(x, 3.0))))
else:
return x * normcdf(x)
```
Linking XLA PR - https://github.com/pytorch/xla/pull/3039
Pull Request resolved: https://github.com/pytorch/pytorch/pull/61439
Reviewed By: mikaylagawarecki
Differential Revision: D33744717
Pulled By: jbschlosser
fbshipit-source-id: d64532a562ed53247bb4fa52bb16722634d5c187
(cherry picked from commit 4713dd9cca)
This commit is contained in:
committed by
PyTorch MergeBot
parent
e58d5b718a
commit
f499ab9cef
@@ -973,10 +973,17 @@ TEST_F(FunctionalTest, GLU) {
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, GELU) {
|
||||
GELU model;
|
||||
const auto x = torch::linspace(-3.0, 3.0, 100);
|
||||
const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0)));
|
||||
const auto y = F::gelu(x);
|
||||
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kNone));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, TanhGELU) {
|
||||
const auto x = torch::linspace(-3.0, 3.0, 100);
|
||||
const auto inner = std::sqrt(2 / M_PI) * (x + 0.044715 * x.pow(3.0));
|
||||
const auto y_exp = 0.5 * x * (1.0 + inner.tanh());
|
||||
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kTanh));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user