mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
[nn] zero_grad() set_to_none default True (#92731)
Attempts to fix #92656 BC-breaking! This changes the default of zero_grad in optim and in nn to default set grads to None instead of zero tensors. We are changing the default because there are proven perf wins and existing code has typically not regressed due to this change. (will probably have to flesh out this note more). Pull Request resolved: https://github.com/pytorch/pytorch/pull/92731 Approved by: https://github.com/ngimel
This commit is contained in:
committed by
PyTorch MergeBot
parent
5441f2c067
commit
b90496eef5
@@ -45,8 +45,7 @@ TEST_F(ModuleTest, ZeroGrad) {
|
||||
for (auto& parameter : module->parameters()) {
|
||||
// NOLINTNEXTLINE(performance-unnecessary-copy-initialization)
|
||||
auto grad = parameter.grad();
|
||||
ASSERT_TRUE(grad.defined());
|
||||
ASSERT_EQ(grad.sum().item<float>(), 0);
|
||||
ASSERT_FALSE(grad.defined());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,14 +65,14 @@ TEST_F(ModuleTest, ZeroGradWithUndefined) {
|
||||
ASSERT_TRUE(module.x.grad().defined());
|
||||
ASSERT_FALSE(module.y.grad().defined());
|
||||
|
||||
module.zero_grad();
|
||||
module.zero_grad(false); // set_to_none = false
|
||||
|
||||
ASSERT_TRUE(module.x.grad().defined());
|
||||
ASSERT_FALSE(module.y.grad().defined());
|
||||
|
||||
ASSERT_EQ(module.x.grad().sum().item<float>(), 0);
|
||||
|
||||
module.zero_grad(true); // set_to_none = true
|
||||
module.zero_grad();
|
||||
|
||||
ASSERT_FALSE(module.x.grad().defined());
|
||||
ASSERT_FALSE(module.y.grad().defined());
|
||||
|
||||
Reference in New Issue
Block a user