mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Enable BFloat16 ELU, SELU and CELU in CPU path (#62546)
Enable BFloat16 ELU, SELU and CELU in CPU path. SELU and CELU will call ELU implementation. Pull Request resolved: https://github.com/pytorch/pytorch/pull/62546 Approved by: https://github.com/frank-wei
This commit is contained in:
committed by
PyTorch MergeBot
parent
bdbb7fe37a
commit
4f82f439d1
@@ -917,15 +917,21 @@ TEST_F(FunctionalTest, ELU) {
|
||||
for (const auto alpha : {0.0, 0.42, 1.0, 4.2, 42.42}) {
|
||||
auto x = torch::linspace(-10.0, 10.0, size * size * size);
|
||||
x.resize_({size, size, size});
|
||||
auto x_bf16 = torch::linspace(-10.0, 10.0, size * size * size).to(torch::kBFloat16);
|
||||
x_bf16.resize_({size, size, size});
|
||||
|
||||
auto y_exp = torch::max(torch::zeros_like(x), x) +
|
||||
torch::min(torch::zeros_like(x), alpha * (torch::exp(x) - 1.0));
|
||||
auto y = F::elu(x, F::ELUFuncOptions().alpha(alpha).inplace(inplace));
|
||||
auto y_bf16 = F::elu(x_bf16, F::ELUFuncOptions().alpha(alpha).inplace(inplace));
|
||||
|
||||
ASSERT_EQ(y.ndimension(), 3);
|
||||
ASSERT_EQ(y.sizes(), std::vector<int64_t>({size, size, size}));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp));
|
||||
ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2));
|
||||
if (inplace) {
|
||||
ASSERT_TRUE(torch::allclose(x, y_exp));
|
||||
ASSERT_TRUE(torch::allclose(x_bf16.to(torch::kFloat), y, 1e-2, 1e-2));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -938,15 +944,19 @@ TEST_F(FunctionalTest, SELU) {
|
||||
const double alpha = 1.6732632423543772848170429916717;
|
||||
for (const auto inplace : {false, true}) {
|
||||
auto input = torch::randn({5, 5});
|
||||
auto input_bf16 = input.clone().to(torch::kBFloat16);
|
||||
auto expected = scale *
|
||||
(torch::max(torch::zeros_like(input), input) +
|
||||
torch::min(
|
||||
torch::zeros_like(input), alpha * (torch::exp(input) - 1)));
|
||||
auto output = F::selu(input, inplace);
|
||||
auto output_bf16 = F::selu(input_bf16, inplace);
|
||||
|
||||
ASSERT_TRUE(output.allclose(expected));
|
||||
ASSERT_TRUE(output_bf16.to(torch::kFloat).allclose(output, 1e-2, 1e-2));
|
||||
if (inplace) {
|
||||
ASSERT_TRUE(input.allclose(expected));
|
||||
ASSERT_TRUE(input_bf16.to(torch::kFloat).allclose(output, 1e-2, 1e-2));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1535,15 +1545,19 @@ TEST_F(FunctionalTest, CELU) {
|
||||
for (const auto alpha : {0.42, 1.0, 4.2, 42.42}) {
|
||||
auto x = torch::linspace(-10.0, 10.0, size * size * size);
|
||||
x.resize_({size, size, size});
|
||||
auto x_bf16 = x.clone().to(torch::kBFloat16);
|
||||
auto y_exp = torch::max(torch::zeros_like(x), x) +
|
||||
torch::min(torch::zeros_like(x), alpha * (torch::exp(x / alpha) - 1.0));
|
||||
auto y = F::celu(x, F::CELUFuncOptions().alpha(alpha).inplace(inplace));
|
||||
auto y_bf16 = F::celu(x_bf16, F::CELUFuncOptions().alpha(alpha).inplace(inplace));
|
||||
|
||||
ASSERT_EQ(y.ndimension(), 3);
|
||||
ASSERT_EQ(y.sizes(), std::vector<int64_t>({size, size, size}));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp));
|
||||
ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2));
|
||||
if (inplace) {
|
||||
ASSERT_TRUE(torch::allclose(x, y_exp));
|
||||
ASSERT_TRUE(torch::allclose(x_bf16.to(torch::kFloat), y, 1e-2, 1e-2));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1555,13 +1569,16 @@ TEST_F(FunctionalTest, CELUDefaultOptions) {
|
||||
const auto alpha = 1.0;
|
||||
auto x = torch::linspace(-10.0, 10.0, size * size * size);
|
||||
x.resize_({size, size, size});
|
||||
auto x_bf16 = x.clone().to(torch::kBFloat16);
|
||||
auto y_exp = torch::max(torch::zeros_like(x), x) +
|
||||
torch::min(torch::zeros_like(x), alpha * (torch::exp(x / alpha) - 1.0));
|
||||
auto y = F::celu(x);
|
||||
auto y_bf16 = F::celu(x_bf16);
|
||||
|
||||
ASSERT_EQ(y.ndimension(), 3);
|
||||
ASSERT_EQ(y.sizes(), std::vector<int64_t>({size, size, size}));
|
||||
ASSERT_TRUE(torch::allclose(y, y_exp));
|
||||
ASSERT_TRUE(torch::allclose(y_bf16.to(torch::kFloat), y, 1e-2, 1e-2));
|
||||
}
|
||||
|
||||
TEST_F(FunctionalTest, PixelShuffle) {
|
||||
|
||||
Reference in New Issue
Block a user