Add torch.nn.GELU for GELU activation (#28944)

Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/28944

Add torch.nn.GELU for GELU activation

Test Plan: buck test mode/dev-nosan //caffe2/test:nn -- "GELU"

Reviewed By: hl475, houseroad

Differential Revision: D18240946

fbshipit-source-id: 6284b30def9bd4c12bf7fb2ed08b1b2f0310bb78
This commit is contained in:
Xiaomeng Yang
2019-11-03 21:53:32 -08:00
committed by Facebook Github Bot
parent 3bffb730b6
commit 2460dced8f
23 changed files with 304 additions and 190 deletions
+8 -10
View File
@@ -645,6 +645,14 @@ TEST_F(FunctionalTest, SELU) {
}
}
TEST_F(FunctionalTest, GELU) {
GELU model;
const auto x = torch::linspace(-3.0, 3.0, 100);
const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0)));
const auto y = F::gelu(x);
ASSERT_TRUE(torch::allclose(y, y_exp));
}
TEST_F(FunctionalTest, Hardshrink) {
const auto size = 3;
for (const auto lambda : {-4.2, -1.0, -0.42, 0.0, 0.42, 1.0, 4.2, 42.42}) {
@@ -1019,16 +1027,6 @@ TEST_F(FunctionalTest, Normalize) {
}
}
TEST_F(FunctionalTest, GeLU) {
auto x = torch::tensor({{2., 3.}, {4., 5.}});
auto y_exp = torch::tensor({{1.9545, 2.9960}, {3.9999, 5.0000}}) +
torch::tensor({{-2.3842e-07, -4.9829e-05}, {-2.6703e-05, -1.4305e-06}});
auto y = F::gelu(x);
ASSERT_EQ(y.ndimension(), 2);
ASSERT_EQ(y.sizes(), std::vector<int64_t>({2, 2}));
ASSERT_TRUE(torch::allclose(y, y_exp));
}
TEST_F(FunctionalTest, ReLU) {
const auto size = 3;
for (const auto inplace : {false, true}) {