[nn] zero_grad() set_to_none default True (#92731)

Attempts to fix #92656

BC-breaking! This changes the default of zero_grad in optim and in nn to default set grads to None instead of zero tensors. We are changing the default because there are proven perf wins and existing code has typically not regressed due to this change. (will probably have to flesh out this note more).

Pull Request resolved: https://github.com/pytorch/pytorch/pull/92731
Approved by: https://github.com/ngimel
This commit is contained in:
Jane Xu
2023-01-25 19:47:57 +00:00
committed by PyTorch MergeBot
parent 5441f2c067
commit b90496eef5
14 changed files with 48 additions and 46 deletions
+3 -4
View File
@@ -45,8 +45,7 @@ TEST_F(ModuleTest, ZeroGrad) {
for (auto& parameter : module->parameters()) {
// NOLINTNEXTLINE(performance-unnecessary-copy-initialization)
auto grad = parameter.grad();
ASSERT_TRUE(grad.defined());
ASSERT_EQ(grad.sum().item<float>(), 0);
ASSERT_FALSE(grad.defined());
}
}
@@ -66,14 +65,14 @@ TEST_F(ModuleTest, ZeroGradWithUndefined) {
ASSERT_TRUE(module.x.grad().defined());
ASSERT_FALSE(module.y.grad().defined());
module.zero_grad();
module.zero_grad(false); // set_to_none = false
ASSERT_TRUE(module.x.grad().defined());
ASSERT_FALSE(module.y.grad().defined());
ASSERT_EQ(module.x.grad().sum().item<float>(), 0);
module.zero_grad(true); // set_to_none = true
module.zero_grad();
ASSERT_FALSE(module.x.grad().defined());
ASSERT_FALSE(module.y.grad().defined());