diff --git a/test/cpp/api/optim.cpp b/test/cpp/api/optim.cpp index 3c89dd6487f5..68da81fd0512 100644 --- a/test/cpp/api/optim.cpp +++ b/test/cpp/api/optim.cpp @@ -174,30 +174,66 @@ TEST(OptimTest, OptimizerAccessors) { optimizer_.state(); } -TEST(OptimTest, BasicInterface) { +#define OLD_INTERFACE_WARNING_CHECK(func) \ +{ \ + std::stringstream buffer;\ + torch::test::CerrRedirect cerr_redirect(buffer.rdbuf());\ + func;\ + ASSERT_EQ(\ + torch::test::count_substr_occurrences(\ + buffer.str(),\ + "will be removed"\ + ),\ + 1);\ +} + +TEST(OptimTest, OldInterface) { + struct MyOptimizerOptions : public OptimizerCloneableOptions { + MyOptimizerOptions(double lr = 1.0) : lr_(lr) {}; + TORCH_ARG(double, lr) = 1.0; + }; + struct MyOptimizer : Optimizer { using Optimizer::Optimizer; torch::Tensor step(LossClosure closure = nullptr) override { return {};} + explicit MyOptimizer( + std::vector params, MyOptimizerOptions defaults = {}) : + Optimizer({std::move(OptimizerParamGroup(params))}, std::make_unique(defaults)) {} }; std::vector parameters = { torch::ones({2, 3}), torch::zeros({2, 3}), torch::rand({2, 3})}; { MyOptimizer optimizer(parameters); - ASSERT_EQ(optimizer.size(), parameters.size()); + size_t size; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, parameters.size()); } { - MyOptimizer optimizer; - ASSERT_EQ(optimizer.size(), 0); - optimizer.add_parameters(parameters); - ASSERT_EQ(optimizer.size(), parameters.size()); - for (size_t p = 0; p < parameters.size(); ++p) { - ASSERT_TRUE(optimizer.parameters()[p].allclose(parameters[p])); + std::vector params; + MyOptimizer optimizer(params); + + size_t size; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, 0); + + OLD_INTERFACE_WARNING_CHECK(optimizer.add_parameters(parameters)); + + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, parameters.size()); + + std::vector params_; + OLD_INTERFACE_WARNING_CHECK(params_ = optimizer.parameters()); + for (size_t p = 0; p < size; ++p) { + ASSERT_TRUE(params_[p].allclose(parameters[p])); } } { Linear linear(3, 4); MyOptimizer optimizer(linear->parameters()); - ASSERT_EQ(optimizer.size(), linear->parameters().size()); + + size_t size; + OLD_INTERFACE_WARNING_CHECK(size = optimizer.size()); + ASSERT_EQ(size, linear->parameters().size()); } } @@ -388,7 +424,7 @@ TEST(OptimTest, AddParameter_LBFGS) { } LBFGS optimizer(std::vector{}, 1.0); - optimizer.add_parameters(parameters); + OLD_INTERFACE_WARNING_CHECK(optimizer.add_parameters(parameters)); optimizer.step([]() { return torch::tensor(1); }); diff --git a/test/cpp/api/serialize.cpp b/test/cpp/api/serialize.cpp index b69e2927e16a..2d859ffc253f 100644 --- a/test/cpp/api/serialize.cpp +++ b/test/cpp/api/serialize.cpp @@ -64,7 +64,7 @@ void is_optimizer_state_equal( } template -void test_serialize_optimizer(DerivedOptimizerOptions options) { +void test_serialize_optimizer(DerivedOptimizerOptions options, bool only_has_global_state = false) { auto model1 = Linear(5, 2); auto model2 = Linear(5, 2); auto model3 = Linear(5, 2); @@ -125,9 +125,11 @@ void test_serialize_optimizer(DerivedOptimizerOptions options) { auto& optim3_2_state = optim3_2.state(); auto& optim3_state = optim3.state(); - // optim3_2 and optim1 should have param_groups and state of size 1 and 2 respectively + // optim3_2 and optim1 should have param_groups and state of size 1 and state_size respectively ASSERT_TRUE(optim3_2_param_groups.size() == 1); - ASSERT_TRUE(optim3_2_state.size() == 2); + // state_size = 2 for all optimizers except LBFGS as LBFGS only maintains one global state + int state_size = only_has_global_state ? 1 : 2; + ASSERT_TRUE(optim3_2_state.size() == state_size); // optim3_2 and optim1 should have param_groups and state of same size ASSERT_TRUE(optim3_2_param_groups.size() == optim3_param_groups.size()); @@ -668,39 +670,16 @@ TEST(SerializeTest, Optim_RMSprop) { } TEST(SerializeTest, Optim_LBFGS) { - auto options = LBFGSOptions(); + test_serialize_optimizer(LBFGSOptions(), true); + // bc compatibility check auto model1 = Linear(5, 2); - auto model2 = Linear(5, 2); - auto model3 = Linear(5, 2); - - // Models 1, 2, 3 will have the same parameters. - auto model_tempfile = c10::make_tempfile(); - torch::save(model1, model_tempfile.name); - torch::load(model2, model_tempfile.name); - torch::load(model3, model_tempfile.name); - - auto param1 = model1->named_parameters(); - auto param2 = model2->named_parameters(); - auto param3 = model3->named_parameters(); - for (const auto& p : param1) { - ASSERT_TRUE(p->allclose(param2[p.key()])); - ASSERT_TRUE(param2[p.key()].allclose(param3[p.key()])); - } - // Make some optimizers - auto optim1 = LBFGS( - {torch::optim::OptimizerParamGroup(model1->parameters())}, options); - auto optim2 = LBFGS( - model2->parameters(), options); - auto optim2_2 = LBFGS( - model2->parameters(), options); - auto optim3 = LBFGS( - model3->parameters(), options); - auto optim3_2 = LBFGS( - model3->parameters(), options); + auto model1_params = model1->parameters(); + // added a tensor for lazy init check - when all params do not have entry in buffers + model1_params.emplace_back(torch::randn({2,3})); + auto optim1 = torch::optim::LBFGS(model1_params, torch::optim::LBFGSOptions()); auto x = torch::ones({10, 5}); - - auto step = [&x](torch::optim::LossClosureOptimizer& optimizer, Linear model) { + auto step = [&x](torch::optim::Optimizer& optimizer, Linear model) { optimizer.zero_grad(); auto y = model->forward(x).sum(); y.backward(); @@ -708,56 +687,47 @@ TEST(SerializeTest, Optim_LBFGS) { optimizer.step(closure); }; - // Do 2 steps of model1 - step(optim1, model1); step(optim1, model1); - // Do 2 steps of model 2 without saving the optimizer - step(optim2, model2); - step(optim2_2, model2); + at::Tensor d, t, H_diag, prev_flat_grad, prev_loss; + std::deque old_dirs, old_stps; - // Do 1 step of model 3 - step(optim3, model3); + const auto& params_ = optim1.param_groups()[0].params(); + auto key_ = c10::guts::to_string(params_[0].unsafeGetTensorImpl()); + const auto& optim1_state = static_cast(*(optim1.state().at(key_).get())); + d = optim1_state.d(); + t = at::tensor(optim1_state.t()); + H_diag = optim1_state.H_diag(); + prev_flat_grad = optim1_state.prev_flat_grad(); + prev_loss = at::tensor(optim1_state.prev_loss()); + old_dirs = optim1_state.old_dirs(); - // save the optimizer - auto optim_tempfile = c10::make_tempfile(); - torch::save(optim3, optim_tempfile.name); - torch::load(optim3_2, optim_tempfile.name); + // write buffers to the file + auto optim_tempfile_old_format = c10::make_tempfile(); + torch::serialize::OutputArchive output_archive; + output_archive.write("d", d, /*is_buffer=*/true); + output_archive.write("t", t, /*is_buffer=*/true); + output_archive.write("H_diag", H_diag, /*is_buffer=*/true); + output_archive.write("prev_flat_grad", prev_flat_grad, /*is_buffer=*/true); + output_archive.write("prev_loss", prev_loss, /*is_buffer=*/true); + write_tensors_to_archive(output_archive, "old_dirs", old_dirs); + write_tensors_to_archive(output_archive, "old_stps", old_stps); + output_archive.save_to(optim_tempfile_old_format.name); - auto& optim3_2_param_groups = optim3_2.param_groups(); - auto& optim3_param_groups = optim3.param_groups(); - auto& optim3_2_state = optim3_2.state(); - auto& optim3_state = optim3.state(); + auto optim1_2 = LBFGS(model1_params, torch::optim::LBFGSOptions()); + OLD_SERIALIZATION_LOGIC_WARNING_CHECK(torch::load, optim1_2, optim_tempfile_old_format.name); - // LBFGS only supports 1 param_group - // optim3_2 and optim1 should have param_groups of size 1 - ASSERT_TRUE(optim3_param_groups.size() == 1); - ASSERT_TRUE(optim3_2_param_groups.size() == 1); - // LBFGS only maintains one global state - ASSERT_TRUE(optim3_2_state.size() == 1); - ASSERT_TRUE(optim3_state.size() == 1); + const auto& params1_2_ = optim1_2.param_groups()[0].params(); + auto param_key = c10::guts::to_string(params1_2_[0].unsafeGetTensorImpl()); + auto& optim1_2_state = static_cast(*(optim1_2.state().at(param_key).get())); - // checking correctness of serialization logic for optimizer.param_groups_ and optimizer.state_ - for (int i = 0; i < optim3_2_param_groups.size(); i++) { - is_optimizer_param_group_equal( - optim3_2_param_groups[i], optim3_param_groups[i]); - is_optimizer_state_equal(optim3_2_state, optim3_state); - } + // old LBFGS didn't track func_evals, n_iter, ro, al values + optim1_2_state.func_evals(optim1_state.func_evals()); + optim1_2_state.n_iter(optim1_state.n_iter()); + optim1_2_state.ro(optim1_state.ro()); + optim1_2_state.al(optim1_state.al()); - // Do step2 for model 3 - step(optim3_2, model3); - - param1 = model1->named_parameters(); - param2 = model2->named_parameters(); - param3 = model3->named_parameters(); - for (const auto& p : param1) { - const auto& name = p.key(); - // Model 1 and 3 should be the same - ASSERT_TRUE( - param1[name].norm().item() == param3[name].norm().item()); - ASSERT_TRUE( - param1[name].norm().item() != param2[name].norm().item()); - } + is_optimizer_state_equal(optim1.state(), optim1_2.state()); } TEST(SerializeTest, XOR_CUDA) { diff --git a/torch/csrc/api/include/torch/optim/adagrad.h b/torch/csrc/api/include/torch/optim/adagrad.h index 0860bbf13fab..bd05cb424d1f 100644 --- a/torch/csrc/api/include/torch/optim/adagrad.h +++ b/torch/csrc/api/include/torch/optim/adagrad.h @@ -47,7 +47,7 @@ public: class TORCH_API Adagrad : public Optimizer { public: explicit Adagrad(std::vector param_groups, - AdagradOptions defaults) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { + AdagradOptions defaults = {}) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); TORCH_CHECK(defaults.lr_decay() >= 0, "Invalid lr_decay value: ", defaults.lr_decay()); TORCH_CHECK(defaults.weight_decay() >= 0, "Invalid weight_decay value: ", defaults.weight_decay()); @@ -66,22 +66,9 @@ class TORCH_API Adagrad : public Optimizer { explicit Adagrad( std::vector params, - AdagradOptions defaults) : Adagrad({std::move(OptimizerParamGroup(params))}, defaults) {} + AdagradOptions defaults = {}) : Adagrad({std::move(OptimizerParamGroup(params))}, defaults) {} torch::Tensor step(LossClosure closure = nullptr) override; - - /// Adds the given vector of parameters to the optimizer's parameter list. - void add_parameters(const std::vector& parameters) override; - - /// Provides a const reference to the parameters this optimizer holds. - const std::vector& parameters() const noexcept override; - - /// Provides a reference to the parameters this optimizer holds. - std::vector& parameters() noexcept override; - - /// Returns the number of parameters referenced by the optimizer. - size_t size() const noexcept override; - void save(serialize::OutputArchive& archive) const override; void load(serialize::InputArchive& archive) override; diff --git a/torch/csrc/api/include/torch/optim/adam.h b/torch/csrc/api/include/torch/optim/adam.h index 8957dfb5c7a2..6878b725f2bb 100644 --- a/torch/csrc/api/include/torch/optim/adam.h +++ b/torch/csrc/api/include/torch/optim/adam.h @@ -49,7 +49,7 @@ public: class TORCH_API Adam : public Optimizer { public: explicit Adam(std::vector param_groups, - AdamOptions defaults) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { + AdamOptions defaults = {}) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); auto betas = defaults.betas(); @@ -59,17 +59,12 @@ class TORCH_API Adam : public Optimizer { } explicit Adam( std::vector params, - AdamOptions defaults) : Adam({std::move(OptimizerParamGroup(params))}, defaults) {} + AdamOptions defaults = {}) : Adam({std::move(OptimizerParamGroup(params))}, defaults) {} torch::Tensor step(LossClosure closure = nullptr) override; void save(serialize::OutputArchive& archive) const override; void load(serialize::InputArchive& archive) override; - void add_parameters(const std::vector& parameters) override; - const std::vector& parameters() const noexcept override; - std::vector& parameters() noexcept override; - size_t size() const noexcept override; - private: template static void serialize(Self& self, Archive& archive) { diff --git a/torch/csrc/api/include/torch/optim/lbfgs.h b/torch/csrc/api/include/torch/optim/lbfgs.h index d2c9e365dafc..6d96a61efc3b 100644 --- a/torch/csrc/api/include/torch/optim/lbfgs.h +++ b/torch/csrc/api/include/torch/optim/lbfgs.h @@ -50,10 +50,10 @@ struct TORCH_API LBFGSParamState : public OptimizerCloneableParamState param_groups, - LBFGSOptions defaults) : LossClosureOptimizer(std::move(param_groups), std::make_unique(defaults)) { + LBFGSOptions defaults = {}) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { TORCH_CHECK(param_groups_.size() == 1, "LBFGS doesn't support per-parameter options (parameter groups)"); if (defaults.max_eval() == c10::nullopt) { auto max_eval_val = (defaults.max_iter() * 5) / 4; @@ -64,13 +64,9 @@ class TORCH_API LBFGS : public LossClosureOptimizer { } explicit LBFGS( std::vector params, - LBFGSOptions defaults) : LBFGS({std::move(OptimizerParamGroup(params))}, defaults) {} + LBFGSOptions defaults = {}) : LBFGS({std::move(OptimizerParamGroup(params))}, defaults) {} Tensor step(LossClosure closure) override; - void add_parameters(const std::vector& parameters) override; - const std::vector& parameters() const noexcept override; - std::vector& parameters() noexcept override; - size_t size() const noexcept override; void save(serialize::OutputArchive& archive) const override; void load(serialize::InputArchive& archive) override; diff --git a/torch/csrc/api/include/torch/optim/optimizer.h b/torch/csrc/api/include/torch/optim/optimizer.h index 0d0c52bda026..d241108d2875 100644 --- a/torch/csrc/api/include/torch/optim/optimizer.h +++ b/torch/csrc/api/include/torch/optim/optimizer.h @@ -40,7 +40,7 @@ class TORCH_API OptimizerParamState { }; template -class TORCH_API OptimizerCloneableParamState : public OptimizerParamState { +class OptimizerCloneableParamState : public OptimizerParamState { std::unique_ptr clone() const override { return std::make_unique(static_cast(*this)); } @@ -55,7 +55,7 @@ class TORCH_API OptimizerOptions { }; template -class TORCH_API OptimizerCloneableOptions : public OptimizerOptions { +class OptimizerCloneableOptions : public OptimizerOptions { std::unique_ptr clone() const override { return std::make_unique(static_cast(*this)); } @@ -81,58 +81,45 @@ class TORCH_API OptimizerParamGroup { std::unique_ptr options_; }; -namespace detail { - -/// Base class for all optimizers, that does not yet define a `step()` -/// mechanism. All it specifies is that optimizers must be supplied with a -/// vector of parameters. It also defines certain methods that all optimizers -/// shall have, such as `zero_grad`. -class TORCH_API OptimizerBase { +class TORCH_API Optimizer { public: // The copy constructor is deleted, because the user should use the // `state_dict` / `load_state_dict` API to copy an optimizer instead. - OptimizerBase(const OptimizerBase& optimizer_base) = delete; - OptimizerBase(OptimizerBase&& optimizer_base) = default; + Optimizer(const Optimizer& optimizer) = delete; + Optimizer(Optimizer&& optimizer) = default; - /// Constructs the `Optimizer` from a vector of parameters. - explicit OptimizerBase(std::vector parameters); - - explicit OptimizerBase(std::vector param_groups, std::unique_ptr defaults) : defaults_(std::move(defaults)) { + explicit Optimizer(std::vector param_groups, std::unique_ptr defaults) : defaults_(std::move(defaults)) { for (const auto& param_group : param_groups) { add_param_group(param_group); } } + /// Constructs the `Optimizer` from a vector of parameters. + explicit Optimizer(std::vector parameters, std::unique_ptr defaults) : Optimizer({std::move(OptimizerParamGroup(parameters))}, std::move(defaults)) {}; + /// Adds the given param_group to the optimizer's param_group list. void add_param_group(const OptimizerParamGroup& param_group); - virtual ~OptimizerBase() = default; + virtual ~Optimizer() = default; - // TODO: when all optimizers use the new design, we can devirtualize some of the following methods - // such as add_parameters() / parameters() / size() + using LossClosure = std::function; + /// A loss function closure, which is expected to return the loss value. + virtual Tensor step(LossClosure closure = nullptr) = 0; /// Adds the given vector of parameters to the optimizer's parameter list. - virtual void add_parameters(const std::vector& parameters); - - virtual void _add_parameters_new_design(const std::vector& parameters); + void add_parameters(const std::vector& parameters); /// Zeros out the gradients of all parameters. - virtual void zero_grad(); + void zero_grad(); - /// Provides a const reference to the parameters this optimizer holds. - virtual const std::vector& parameters() const noexcept; + /// Provides a const reference to the parameters in the first param_group this optimizer holds. + const std::vector& parameters() const noexcept; - virtual const std::vector& _parameters_new_design() const noexcept; - - /// Provides a reference to the parameters this optimizer holds. - virtual std::vector& parameters() noexcept; - - virtual std::vector& _parameters_new_design() noexcept; + /// Provides a reference to the parameters in the first param_group this optimizer holds. + std::vector& parameters() noexcept; /// Returns the number of parameters referenced by the optimizer. - virtual size_t size() const noexcept; - - virtual size_t _size_new_design() const noexcept; + size_t size() const noexcept; OptimizerOptions& defaults() noexcept; @@ -160,27 +147,6 @@ class TORCH_API OptimizerBase { std::vector param_groups_; ska::flat_hash_map> state_; std::unique_ptr defaults_; - OptimizerBase() = default; - - /// Accesses a buffer at the given index. - /// Additionally, zeros out the buffers when this is called on the index - template - T& buffer_at(std::vector& buffers, size_t index) { - if (buffers.size() <= index) { - const auto old_size = buffers.size(); - buffers.resize(index + 1); - std::fill(buffers.begin() + old_size, buffers.end(), T{0}); - } - return buffers[index]; - } - - /// Accesses a buffer at the given index, converts it to the type of the - /// parameter at the corresponding index (a no-op if they match). - /// Additionally, zeros out the buffers when this is called on the index - Tensor& buffer_at(std::vector& buffers, size_t index); - - /// The parameters this optimizer optimizes. - std::vector parameters_; }; /* How do we decide whether to serialize undefined tensors or @@ -201,39 +167,15 @@ b) For c10::nullopt value: in param state, c10::nullopt value in C++ impl is equ missing key in Python impl. Since we don't serialize missing keys in Python API, we skip c10::nullopt values when serializing the param state. */ -/// Serializes an `OptimizerBase` into an `OutputArchive`. +/// Serializes an `Optimizer` into an `OutputArchive`. TORCH_API serialize::OutputArchive& operator<<( serialize::OutputArchive& archive, - const OptimizerBase& optimizer); + const Optimizer& optimizer); /// Deserializes a `Tensor` from an `InputArchive`. TORCH_API serialize::InputArchive& operator>>( serialize::InputArchive& archive, - OptimizerBase& optimizer); -} // namespace detail - -/// Optimizer that can optionally take a loss function in `step()` method -/// and returns the loss value. The only side effect is that parameters are updated -/// according to the concrete optimization algorithm. -class Optimizer : public detail::OptimizerBase { - public: - /// A loss function closure, which is expected to return the loss value. - using LossClosure = std::function; - using detail::OptimizerBase::OptimizerBase; - virtual Tensor step(LossClosure closure = nullptr) = 0; -}; - -/// Optimizer that requires the loss function to be supplied to the `step()` -/// function, as it may evaluate the loss function multiple times per step. -/// Examples of such algorithms are conjugate gradient and LBFGS. The `step()` -/// function also returns the loss value. -class LossClosureOptimizer : public detail::OptimizerBase { - public: - /// A loss function closure, which is expected to return the loss value. - using LossClosure = std::function; - using detail::OptimizerBase::OptimizerBase; - virtual Tensor step(LossClosure closure) = 0; -}; + Optimizer& optimizer); } // namespace optim } // namespace torch diff --git a/torch/csrc/api/include/torch/optim/rmsprop.h b/torch/csrc/api/include/torch/optim/rmsprop.h index 0f1d5c8f3300..c8a7f139066d 100644 --- a/torch/csrc/api/include/torch/optim/rmsprop.h +++ b/torch/csrc/api/include/torch/optim/rmsprop.h @@ -54,7 +54,7 @@ struct TORCH_API RMSpropParamState : public OptimizerCloneableParamState param_groups, - RMSpropOptions defaults) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { + RMSpropOptions defaults = {}) : Optimizer(std::move(param_groups), std::make_unique(defaults)) { TORCH_CHECK(defaults.lr() >= 0, "Invalid learning rate: ", defaults.lr()); TORCH_CHECK(defaults.eps() >= 0, "Invalid epsilon value: ", defaults.eps()); TORCH_CHECK(defaults.momentum() >= 0, "Invalid momentum value: ", defaults.momentum()); @@ -63,22 +63,9 @@ class TORCH_API RMSprop : public Optimizer { } explicit RMSprop(std::vector params, - RMSpropOptions defaults) : RMSprop({std::move(OptimizerParamGroup(params))}, defaults) {} + RMSpropOptions defaults = {}) : RMSprop({std::move(OptimizerParamGroup(params))}, defaults) {} torch::Tensor step(LossClosure closure = nullptr) override; - - /// Returns the number of parameters referenced by the optimizer. - size_t size() const noexcept override; - - /// Adds the given vector of parameters to the optimizer's parameter list. - void add_parameters(const std::vector& parameters) override; - - /// Provides a const reference to the parameters this optimizer holds. - const std::vector& parameters() const noexcept override; - - /// Provides a reference to the parameters this optimizer holds. - std::vector& parameters() noexcept override; - void save(serialize::OutputArchive& archive) const override; void load(serialize::InputArchive& archive) override; diff --git a/torch/csrc/api/include/torch/optim/serialize.h b/torch/csrc/api/include/torch/optim/serialize.h index 671011c4afc8..72344932295a 100644 --- a/torch/csrc/api/include/torch/optim/serialize.h +++ b/torch/csrc/api/include/torch/optim/serialize.h @@ -131,7 +131,7 @@ void serialize( template void serialize( serialize::OutputArchive& archive, - const detail::OptimizerBase& optimizer) { + const Optimizer& optimizer) { archive.write("pytorch_version", IValue("1.5.0")); serialize::OutputArchive state_archive(archive.compilation_unit()); detail::serialize(state_archive, optimizer.state()); @@ -146,7 +146,7 @@ void serialize( template void serialize( serialize::InputArchive& archive, - detail::OptimizerBase& optimizer) { + Optimizer& optimizer) { IValue pytorch_version; archive.read("pytorch_version", pytorch_version); diff --git a/torch/csrc/api/include/torch/optim/sgd.h b/torch/csrc/api/include/torch/optim/sgd.h index dd397c954fff..881dccfec932 100644 --- a/torch/csrc/api/include/torch/optim/sgd.h +++ b/torch/csrc/api/include/torch/optim/sgd.h @@ -22,7 +22,7 @@ namespace torch { namespace optim { struct TORCH_API SGDOptions : public OptimizerCloneableOptions { - /* implicit */ SGDOptions(double lr); + SGDOptions(double lr); TORCH_ARG(double, lr); TORCH_ARG(double, momentum) = 0; TORCH_ARG(double, dampening) = 0; @@ -60,18 +60,6 @@ class TORCH_API SGD : public Optimizer { torch::Tensor step(LossClosure closure = nullptr) override; - /// Adds the given vector of parameters to the optimizer's parameter list. - void add_parameters(const std::vector& parameters) override; - - /// Provides a const reference to the parameters this optimizer holds. - const std::vector& parameters() const noexcept override; - - /// Provides a reference to the parameters this optimizer holds. - std::vector& parameters() noexcept override; - - /// Returns the number of parameters referenced by the optimizer. - size_t size() const noexcept override; - void save(serialize::OutputArchive& archive) const override; void load(serialize::InputArchive& archive) override; diff --git a/torch/csrc/api/src/optim/adagrad.cpp b/torch/csrc/api/src/optim/adagrad.cpp index 187230affd4a..ebc9e2f4b9d0 100644 --- a/torch/csrc/api/src/optim/adagrad.cpp +++ b/torch/csrc/api/src/optim/adagrad.cpp @@ -109,22 +109,6 @@ Tensor Adagrad::step(LossClosure closure) { return loss; } -void Adagrad::add_parameters(const std::vector& parameters) { - return _add_parameters_new_design(parameters); -} - -const std::vector& Adagrad::parameters() const noexcept { - return _parameters_new_design(); -} - -std::vector& Adagrad::parameters() noexcept { - return _parameters_new_design(); -} - -size_t Adagrad::size() const noexcept { - return _size_new_design(); -} - void Adagrad::save(serialize::OutputArchive& archive) const { serialize(*this, archive); } diff --git a/torch/csrc/api/src/optim/adam.cpp b/torch/csrc/api/src/optim/adam.cpp index 73e44236a30d..ccbae6a51106 100644 --- a/torch/csrc/api/src/optim/adam.cpp +++ b/torch/csrc/api/src/optim/adam.cpp @@ -12,6 +12,7 @@ namespace torch { namespace optim { + AdamOptions::AdamOptions(double lr) : lr_(lr) {} bool operator==(const AdamOptions& lhs, const AdamOptions& rhs) { @@ -129,22 +130,6 @@ Tensor Adam::step(LossClosure closure) { return loss; } -void Adam::add_parameters(const std::vector& parameters) { - return _add_parameters_new_design(parameters); -} - -const std::vector& Adam::parameters() const noexcept { - return _parameters_new_design(); -} - -std::vector& Adam::parameters() noexcept { - return _parameters_new_design(); -} - -size_t Adam::size() const noexcept { - return _size_new_design(); -} - void Adam::save(serialize::OutputArchive& archive) const { serialize(*this, archive); } diff --git a/torch/csrc/api/src/optim/lbfgs.cpp b/torch/csrc/api/src/optim/lbfgs.cpp index ef72738e9316..a313e30c25c9 100644 --- a/torch/csrc/api/src/optim/lbfgs.cpp +++ b/torch/csrc/api/src/optim/lbfgs.cpp @@ -552,22 +552,6 @@ Tensor LBFGS::step(LossClosure closure) { return orig_loss; } -void LBFGS::add_parameters(const std::vector& parameters) { - return _add_parameters_new_design(parameters); -} - -const std::vector& LBFGS::parameters() const noexcept { - return _parameters_new_design(); -} - -std::vector& LBFGS::parameters() noexcept { - return _parameters_new_design(); -} - -size_t LBFGS::size() const noexcept { - return _size_new_design(); -} - void LBFGS::save(serialize::OutputArchive& archive) const { serialize(*this, archive); } diff --git a/torch/csrc/api/src/optim/optimizer.cpp b/torch/csrc/api/src/optim/optimizer.cpp index 7fca24032bd2..756e20a4893f 100644 --- a/torch/csrc/api/src/optim/optimizer.cpp +++ b/torch/csrc/api/src/optim/optimizer.cpp @@ -75,14 +75,11 @@ void OptimizerOptions::serialize(torch::serialize::OutputArchive& archive) const "You must override it in your subclass of torch::optim::OptimizerCloneableOptions."); } -namespace detail { -OptimizerBase::OptimizerBase(std::vector parameters) - : parameters_(std::move(parameters)) {} - -void OptimizerBase::add_param_group(const OptimizerParamGroup& param_group) { +void Optimizer::add_param_group(const OptimizerParamGroup& param_group) { for (const auto& param : param_group.params()) { TORCH_CHECK(param.is_leaf(), "can't optimize a non-leaf Tensor"); } + TORCH_INTERNAL_ASSERT(defaults_ != nullptr); OptimizerParamGroup param_group_(param_group.params()); if (!param_group.has_options()) { param_group_.set_options(defaults_->clone()); @@ -96,22 +93,13 @@ void OptimizerBase::add_param_group(const OptimizerParamGroup& param_group) { param_groups_.emplace_back(std::move(param_group_)); } -void OptimizerBase::add_parameters(const std::vector& parameters) { - parameters_.insert(parameters_.end(), parameters.begin(), parameters.end()); -} - -void OptimizerBase::_add_parameters_new_design(const std::vector& parameters) { +void Optimizer::add_parameters(const std::vector& parameters) { + TORCH_WARN("Optimizer::add_parameters() will be removed in PyTorch 1.6"); auto& parameters_ = param_groups_[0].params(); parameters_.insert(parameters_.end(), parameters.begin(), parameters.end()); } -void OptimizerBase::zero_grad() { - for (auto& parameter : parameters_) { - if (parameter.grad().defined()) { - parameter.grad().detach_(); - parameter.grad().zero_(); - } - } +void Optimizer::zero_grad() { for (auto& group : param_groups_) { for (auto& p : group.params()) { if (p.grad().defined()) { @@ -122,30 +110,18 @@ void OptimizerBase::zero_grad() { } } -// TODO: remove this function after all the optimizers use the new design -const std::vector& OptimizerBase::parameters() const noexcept { - return parameters_; -} - -const std::vector& OptimizerBase::_parameters_new_design() const noexcept { +const std::vector& Optimizer::parameters() const noexcept { + TORCH_WARN("Optimizer::parameters() will be removed in PyTorch 1.6"); return param_groups_.at(0).params(); } -// TODO: remove this function after all the optimizers use the new design -std::vector& OptimizerBase::parameters() noexcept { - return parameters_; -} - -std::vector& OptimizerBase::_parameters_new_design() noexcept { +std::vector& Optimizer::parameters() noexcept { + TORCH_WARN("Optimizer::parameters() will be removed in PyTorch 1.6"); return param_groups_.at(0).params(); } -// TODO: update size to return the sum of #params in all param_groups -size_t OptimizerBase::size() const noexcept { - return parameters_.size(); -} - -size_t OptimizerBase::_size_new_design() const noexcept { +size_t Optimizer::size() const noexcept { + TORCH_WARN("Optimizer::size() will be removed in PyTorch 1.6"); size_t count = 0; for (const auto& group : param_groups_) { count += group.params().size(); @@ -153,54 +129,37 @@ size_t OptimizerBase::_size_new_design() const noexcept { return count; } -OptimizerOptions& OptimizerBase::defaults() noexcept { +OptimizerOptions& Optimizer::defaults() noexcept { return *defaults_.get(); } -const OptimizerOptions& OptimizerBase::defaults() const noexcept { +const OptimizerOptions& Optimizer::defaults() const noexcept { return *defaults_.get(); } -std::vector& OptimizerBase::param_groups() noexcept { +std::vector& Optimizer::param_groups() noexcept { return param_groups_; } -const std::vector& OptimizerBase::param_groups() const noexcept { +const std::vector& Optimizer::param_groups() const noexcept { return param_groups_; } -ska::flat_hash_map>& OptimizerBase::state() noexcept { +ska::flat_hash_map>& Optimizer::state() noexcept { return state_; } -const ska::flat_hash_map>& OptimizerBase::state() const noexcept { +const ska::flat_hash_map>& Optimizer::state() const noexcept { return state_; } -Tensor& OptimizerBase::buffer_at(std::vector& buffers, size_t index) { - if (buffers.size() <= index) { - buffers.reserve(index); - for (auto i = buffers.size(); i <= index; ++i) { - buffers.emplace_back(torch::zeros_like(parameters_.at(i))); - } - } - // Copy the buffer to the device and dtype of the parameter. - const auto& parameter = parameters_.at(index); - const auto& buffer = buffers.at(index); - if (buffer.device() != parameter.device() || - buffer.dtype() != parameter.dtype()) { - buffers[index] = buffer.to(parameter.device(), parameter.scalar_type()); - } - return buffers[index]; -} +void Optimizer::save(serialize::OutputArchive& archive) const {} +void Optimizer::load(serialize::InputArchive& archive) {} -void OptimizerBase::save(serialize::OutputArchive& archive) const {} -void OptimizerBase::load(serialize::InputArchive& archive) {} - -/// Serializes an `OptimizerBase` into an `OutputArchive`. +/// Serializes an `Optimizer` into an `OutputArchive`. serialize::OutputArchive& operator<<( serialize::OutputArchive& archive, - const OptimizerBase& optimizer) { + const Optimizer& optimizer) { optimizer.save(archive); return archive; } @@ -208,10 +167,10 @@ serialize::OutputArchive& operator<<( /// Deserializes a `Tensor` from an `InputArchive`. serialize::InputArchive& operator>>( serialize::InputArchive& archive, - OptimizerBase& optimizer) { + Optimizer& optimizer) { optimizer.load(archive); return archive; } -} // namespace detail + } // namespace optim } // namespace torch diff --git a/torch/csrc/api/src/optim/rmsprop.cpp b/torch/csrc/api/src/optim/rmsprop.cpp index 96732ba54646..f91ca12cdde2 100644 --- a/torch/csrc/api/src/optim/rmsprop.cpp +++ b/torch/csrc/api/src/optim/rmsprop.cpp @@ -129,22 +129,6 @@ Tensor RMSprop::step(LossClosure closure) { return loss; } -void RMSprop::add_parameters(const std::vector& parameters) { - return _add_parameters_new_design(parameters); -} - -const std::vector& RMSprop::parameters() const noexcept { - return _parameters_new_design(); -} - -std::vector& RMSprop::parameters() noexcept { - return _parameters_new_design(); -} - -size_t RMSprop::size() const noexcept { - return _size_new_design(); -} - void RMSprop::save(serialize::OutputArchive& archive) const { serialize(*this, archive); } diff --git a/torch/csrc/api/src/optim/sgd.cpp b/torch/csrc/api/src/optim/sgd.cpp index 5bf7043abccb..67c6c1c0772b 100644 --- a/torch/csrc/api/src/optim/sgd.cpp +++ b/torch/csrc/api/src/optim/sgd.cpp @@ -98,22 +98,6 @@ Tensor SGD::step(LossClosure closure) { return loss; } -void SGD::add_parameters(const std::vector& parameters) { - return _add_parameters_new_design(parameters); -} - -const std::vector& SGD::parameters() const noexcept { - return _parameters_new_design(); -} - -std::vector& SGD::parameters() noexcept { - return _parameters_new_design(); -} - -size_t SGD::size() const noexcept { - return _size_new_design(); -} - void SGD::save(serialize::OutputArchive& archive) const { serialize(*this, archive); }