mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Revert D33744717: [pytorch][PR] Implement Tanh Gelu Approximation
Test Plan: revert-hammer
Differential Revision:
D33744717 (https://github.com/pytorch/pytorch/commit/f499ab9ceff49be0b64a15020103c6baec3387ea)
Original commit changeset: d64532a562ed
Original Phabricator Diff: D33744717 (https://github.com/pytorch/pytorch/commit/f499ab9ceff49be0b64a15020103c6baec3387ea)
fbshipit-source-id: 396c3f63de5865f894dbc353d0790a01a624be93
(cherry picked from commit e9fb2d1db1)
This commit is contained in:
committed by
PyTorch MergeBot
parent
0c3bc426a8
commit
cb823d9f07
@@ -973,17 +973,10 @@ TEST_F(FunctionalTest, GLU) {
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, GELU) {
|
||||
GELU model;
|
||||
const auto x = torch::linspace(-3.0, 3.0, 100);
|
||||
const auto y_exp = x * 0.5 * (1.0 + torch::erf(x / std::sqrt(2.0)));
|
||||
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kNone));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, TanhGELU) {
|
||||
const auto x = torch::linspace(-3.0, 3.0, 100);
|
||||
const auto inner = std::sqrt(2 / M_PI) * (x + 0.044715 * x.pow(3.0));
|
||||
const auto y_exp = 0.5 * x * (1.0 + inner.tanh());
|
||||
const auto y = F::gelu(x, F::GELUFuncOptions().approximate(torch::kTanh));
|
||||
const auto y = F::gelu(x);
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp, 1.4e-06, 1e-05));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user