mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Add torch.nn.GELU for GELU activation (#28944)
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/28944 Add torch.nn.GELU for GELU activation Test Plan: buck test mode/dev-nosan //caffe2/test:nn -- "GELU" Reviewed By: hl475, houseroad Differential Revision: D18240946 fbshipit-source-id: 6284b30def9bd4c12bf7fb2ed08b1b2f0310bb78
This commit is contained in:
committed by
Facebook Github Bot
parent
3bffb730b6
commit
2460dced8f
@@ -645,6 +645,14 @@ TEST_F(FunctionalTest, SELU) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, GELU) {
|
||||
GELU model;
|
||||
const auto x = torch::linspace(-3.0, 3.0, 100);
|
||||
const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0)));
|
||||
const auto y = F::gelu(x);
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp));
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, Hardshrink) {
|
||||
const auto size = 3;
|
||||
for (const auto lambda : {-4.2, -1.0, -0.42, 0.0, 0.42, 1.0, 4.2, 42.42}) {
|
||||
@@ -1019,16 +1027,6 @@ TEST_F(FunctionalTest, Normalize) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, GeLU) {
|
||||
auto x = torch::tensor({{2., 3.}, {4., 5.}});
|
||||
auto y_exp = torch::tensor({{1.9545, 2.9960}, {3.9999, 5.0000}}) +
|
||||
torch::tensor({{-2.3842e-07, -4.9829e-05}, {-2.6703e-05, -1.4305e-06}});
|
||||
auto y = F::gelu(x);
|
||||
ASSERT_EQ(y.ndimension(), 2);
|
||||
ASSERT_EQ(y.sizes(), std::vector<int64_t>({2, 2}));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp));
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, ReLU) {
|
||||
const auto size = 3;
|
||||
for (const auto inplace : {false, true}) {
|
||||
|
||||
Reference in New Issue
Block a user