Revert D33744717: [pytorch][PR] Implement Tanh Gelu Approximation

Test Plan: revert-hammer

Differential Revision:
D33744717 (https://github.com/pytorch/pytorch/commit/f499ab9ceff49be0b64a15020103c6baec3387ea)

Original commit changeset: d64532a562ed

Original Phabricator Diff: D33744717 (https://github.com/pytorch/pytorch/commit/f499ab9ceff49be0b64a15020103c6baec3387ea)

fbshipit-source-id: 396c3f63de5865f894dbc353d0790a01a624be93
(cherry picked from commit e9fb2d1db1)
This commit is contained in:
Joel Schlosser
2022-01-28 10:30:24 -08:00
committed by PyTorch MergeBot
parent 0c3bc426a8
commit cb823d9f07
48 changed files with 199 additions and 749 deletions
+2 -9
View File
@@ -973,17 +973,10 @@ TEST_F(FunctionalTest, GLU) {
}
TEST_F(FunctionalTest, GELU) {
GELU model;
const auto x = torch::linspace(-3.0, 3.0, 100);
const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0)));
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kNone));
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
}
TEST_F(FunctionalTest, TanhGELU) {
const auto x = torch::linspace(-3.0, 3.0, 100);
const auto inner = std::sqrt(2 / M_PI) * (x + 0.044715 * x.pow(3.0));
const auto y_exp = 0.5 * x * (1.0 + inner.tanh());
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kTanh));
const auto y = F::gelu(x);
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
}