InstanceNorm{1,2,3}d (#28790)

Summary:
Hi yf225,

I have a few doubts related to implementation:
1) What tests do I have to write?
2) What does _load_state_from_dict does?
3) Do I need to override reset() function as I can not see it's utility?
4) InstanceNormOptions could be removed with BatchNormOptions, but I find that
`track_running_status` is not defined instead `stateful` is defined.

InstanceNorm{1,2,3}d https://github.com/pytorch/pytorch/issues/25883
Pull Request resolved: https://github.com/pytorch/pytorch/pull/28790

Differential Revision: D18588666

Pulled By: yf225

fbshipit-source-id: bb9b81f01f62c3fc8765fa0ba0716768087ee155
This commit is contained in:
Divyansh Singhvi
2019-11-19 16:53:37 -08:00
committed by Facebook Github Bot
parent 8e3486de81
commit ec52d911bd
15 changed files with 880 additions and 115 deletions
+230
View File
@@ -1579,6 +1579,236 @@ TEST_F(FunctionalTest, BatchNorm3dDefaultOptions) {
ASSERT_TRUE(output.allclose(expected));
}
TEST_F(FunctionalTest, InstanceNorm1d) {
int num_features = 5;
double eps = 1e-05;
double momentum = 0.1;
auto input = torch::arange(40.).view({2, 5, 4});
auto mean = torch::arange(5.);
auto variance = torch::arange(5.);
auto weight = torch::arange((double)num_features);
auto bias = torch::arange((double)num_features);
auto output = F::instance_norm(
input,
F::InstanceNormFuncOptions()
.running_mean(mean)
.running_var(variance)
.weight(weight)
.bias(bias)
.momentum(momentum)
.eps(eps));
auto expected = torch::tensor({{{ 0.0000, 0.0000, 0.0000, 0.0000},
{-0.3416, 0.5528, 1.4472, 2.3416},
{-0.6833, 1.1056, 2.8944, 4.6833},
{-1.0249, 1.6584, 4.3416, 7.0249},
{-1.3665, 2.2112, 5.7888, 9.3665}},
{{ 0.0000, 0.0000, 0.0000, 0.0000},
{-0.3416, 0.5528, 1.4472, 2.3416},
{-0.6833, 1.1056, 2.8944, 4.6833},
{-1.0249, 1.6584, 4.3416, 7.0249},
{-1.3665, 2.2112, 5.7888, 9.3665}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, InstanceNorm1dDefaultOptions) {
auto input = torch::arange(40.).view({2, 5, 4});
auto output = F::instance_norm(input);
auto expected = torch::tensor({{{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416}},
{{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416},
{-1.3416, -0.4472, 0.4472, 1.3416}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, InstanceNorm2d) {
int num_features = 5;
double eps = 1e-05;
double momentum = 0.1;
auto input = torch::arange(2. * num_features * 2 * 2).view({2, num_features, 2, 2});
auto mean = torch::arange((double)num_features);
auto variance = torch::arange((double)num_features);
auto weight = torch::arange((double)num_features);
auto bias = torch::arange((double)num_features);
auto output = F::instance_norm(
input,
F::InstanceNormFuncOptions()
.running_mean(mean)
.running_var(variance)
.weight(weight)
.bias(bias)
.momentum(momentum)
.eps(eps));
auto expected = torch::tensor({{{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}},
{{-0.3416, 0.5528},
{ 1.4472, 2.3416}},
{{-0.6833, 1.1056},
{ 2.8944, 4.6833}},
{{-1.0249, 1.6584},
{ 4.3416, 7.0249}},
{{-1.3665, 2.2112},
{ 5.7888, 9.3665}}},
{{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}},
{{-0.3416, 0.5528},
{ 1.4472, 2.3416}},
{{-0.6833, 1.1056},
{ 2.8944, 4.6833}},
{{-1.0249, 1.6584},
{ 4.3416, 7.0249}},
{{-1.3665, 2.2112},
{ 5.7888, 9.3665}}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, InstanceNorm2dDefaultOptions) {
int num_features = 5;
double eps = 1e-05;
auto input = torch::arange(2. * num_features * 2 * 2).view({2, num_features, 2, 2});
auto output = F::instance_norm(input);
auto expected = torch::tensor({{{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}}},
{{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}},
{{-1.3416, -0.4472},
{ 0.4472, 1.3416}}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, InstanceNorm3d) {
int num_features = 5;
double eps = 1e-05;
double momentum = 0.1;
auto input = torch::arange(2. * num_features * 2 * 2 * 2).view({2, num_features, 2, 2, 2});
auto mean = torch::arange((double)num_features);
auto variance = torch::arange((double)num_features);
auto weight = torch::arange((double)num_features);
auto bias = torch::arange((double)num_features);
auto output = F::instance_norm(
input,
F::InstanceNormFuncOptions()
.running_mean(mean)
.running_var(variance)
.weight(weight)
.bias(bias)
.momentum(momentum)
.eps(eps));
auto expected = torch::tensor({{{{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}},
{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}}},
{{{-0.5275, -0.0911},
{ 0.3453, 0.7818}},
{{ 1.2182, 1.6547},
{ 2.0911, 2.5275}}},
{{{-1.0550, -0.1822},
{ 0.6907, 1.5636}},
{{ 2.4364, 3.3093},
{ 4.1822, 5.0550}}},
{{{-1.5826, -0.2733},
{ 1.0360, 2.3453}},
{{ 3.6547, 4.9640},
{ 6.2733, 7.5826}}},
{{{-2.1101, -0.3644},
{ 1.3814, 3.1271}},
{{ 4.8729, 6.6186},
{ 8.3644, 10.1101}}}},
{{{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}},
{{ 0.0000, 0.0000},
{ 0.0000, 0.0000}}},
{{{-0.5275, -0.0911},
{ 0.3453, 0.7818}},
{{ 1.2182, 1.6547},
{ 2.0911, 2.5275}}},
{{{-1.0550, -0.1822},
{ 0.6907, 1.5636}},
{{ 2.4364, 3.3093},
{ 4.1822, 5.0550}}},
{{{-1.5826, -0.2733},
{ 1.0360, 2.3453}},
{{ 3.6547, 4.9640},
{ 6.2733, 7.5826}}},
{{{-2.1101, -0.3644},
{ 1.3814, 3.1271}},
{{ 4.8729, 6.6186},
{ 8.3644, 10.1101}}}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, InstanceNorm3dDefaultOptions) {
int num_features = 5;
double eps = 1e-05;
auto input = torch::arange(2. * num_features * 2 * 2 * 2).view({2, num_features, 2, 2, 2});
auto output = F::instance_norm(input);
auto expected = torch::tensor({{{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}}},
{{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}},
{{{-1.5275, -1.0911},
{-0.6547, -0.2182}},
{{ 0.2182, 0.6547},
{ 1.0911, 1.5275}}}}});
ASSERT_TRUE(output.allclose(expected, 2e-04));
}
TEST_F(FunctionalTest, Interpolate) {
{
// 1D interpolation