diff --git a/test/cpp/api/init.cpp b/test/cpp/api/init.cpp index 79c74abeebc9..1f4059de02ea 100644 --- a/test/cpp/api/init.cpp +++ b/test/cpp/api/init.cpp @@ -105,7 +105,7 @@ TEST(InitTest, CanInitializeTensorThatRequiresGrad) { ASSERT_THROWS_WITH( tensor.fill_(1), "a leaf Variable that requires grad " - "is being used in an in-place operation"); + "has been used in an in-place operation"); ASSERT_EQ(torch::nn::init::ones_(tensor).sum().item(), 12); } @@ -168,4 +168,4 @@ TEST(InitTest, FanModeLegacyEnum) { FANMODE_ENUM_LEGACY_WARNING_CHECK(kaiming_uniform_, FanIn, "torch::kFanIn") FANMODE_ENUM_LEGACY_WARNING_CHECK(kaiming_uniform_, FanOut, "torch::kFanOut") -} +} \ No newline at end of file diff --git a/test/test_autograd.py b/test/test_autograd.py index 7a8bd8e394c3..927a4c280247 100644 --- a/test/test_autograd.py +++ b/test/test_autograd.py @@ -3472,7 +3472,7 @@ for shape in [(1,), ()]: run_test(grad_mode=True, requires_grad=True, is_view=True, should_raise_tuple=(None, None, None)) # TODO: views require gradients when created in no_grad mode but their grad_fn is not populated - leaf_grad_err = "a leaf Variable that requires grad is being used in an in-place operation." + leaf_grad_err = "a leaf Variable that requires grad has been used in an in-place operation." run_test(grad_mode=False, requires_grad=True, is_view=True, should_raise_tuple=(leaf_grad_err, leaf_grad_err, leaf_grad_err)) run_test(grad_mode=False, requires_grad=False, is_view=True, @@ -3798,32 +3798,6 @@ for shape in [(1,), ()]: c.backward() self.assertEqual(b.grad, torch.tensor([-inf, 0., 0.]), allow_inf=True) - def test_multi_view_methods(self): - # This list should math the PURE_VIEW_FUNCTIONS in `tools/autograd/gen_autograd.py - # It maps a function its arguments for an input of size [3,] - fn_to_test = { - 'split': (2,), - 'split_with_sizes': ((2, 1),), - 'unbind': (0,) - } - - for fn, arg in fn_to_test.items(): - inp = torch.rand(3, dtype=torch.double, requires_grad=True) - - def foo(inp, inplace=False): - x = inp * 2 - x = getattr(x, fn)(*arg) - res = 0. - for i, el in enumerate(x): - if inplace: - el *= 42 - res += (i + 1) * el.sum() - return res - self.assertTrue(gradcheck(foo, (inp,))) - self.assertTrue(gradgradcheck(foo, (inp,))) - self.assertTrue(gradcheck(foo, (inp, True))) - self.assertTrue(gradgradcheck(foo, (inp, True))) - def index_variable(shape, max_indices): if not isinstance(shape, tuple): diff --git a/tools/autograd/derivatives.yaml b/tools/autograd/derivatives.yaml index b4143d38860d..ddea3a7edcc9 100644 --- a/tools/autograd/derivatives.yaml +++ b/tools/autograd/derivatives.yaml @@ -101,14 +101,9 @@ # of one of its input: # - If there is only one differentiable output, this Tensor is marked as a # differentiable view. -# - If there are more than one differentiable output, by default all the views are +# - If there are more than one differentiable output, all the views are # marked as differentiable views and created with allow_rebase_history=false. # Meaning that any inplace operation on it will raise an error. -# - If there are more than one differentiable output, and the function is only doing -# view operation, it can be added to PURE_VIEW_FUNCTIONS in gen_autograd.py. In that -# case, the outputs won't be created with allow_rebase_history=false but the backward -# specified in this file might NOT be used. See the note in gen_autograd.py for -# more details. # # NB: The parameter names here MUST be consistent with the parameter names # in Decalarations.yaml diff --git a/tools/autograd/gen_autograd.py b/tools/autograd/gen_autograd.py index 6fb503679f46..21a929855c98 100644 --- a/tools/autograd/gen_autograd.py +++ b/tools/autograd/gen_autograd.py @@ -30,7 +30,9 @@ from collections import defaultdict from .utils import YamlLoader, split_name_params # See NOTE [ Autograd View Variables ] in variable.h for details. -# A map: function name => name of the argument that all outputs are view of +# A map: function name => two options: +# 1. name of the argument that all outputs are view of +# 2. map: output idx => name of the argument that this result is view of VIEW_FUNCTIONS = { 'numpy_T': 'self', 'alias': 'self', @@ -66,19 +68,6 @@ VIEW_FUNCTIONS = { # returns a view of its inputs RETURNS_VIEWS_OF_INPUT = set(VIEW_FUNCTIONS.keys()).union({'chunk', 'narrow'}) -# note: For functions that return multiple views, we make a distinction between the -# functions that only perform viewing (called pure views here) versus all others (like -# user-defined Functions). This list is used to know which functions are pure views. -# A function should NOT be added to this list if: -# - Your backward function does more than just the backward of the views. For example if your -# function makes changes to the layout both in forward and backward. -# A function should be added to this list if: -# - You want to allow inplace modification of the result. -# - You are aware that inplace modification of the output will change the memory usage of the -# backward as the multiple Nodes we create will force the gradient buffer to be created -# for each Node, instead of once in the original multi-output Node. -# If you update this list, please update "test_multi_view_methods" in test_autograd.py -PURE_VIEW_FUNCTIONS = ['split', 'split_with_sizes', 'unbind'] def format_return_type(returns): if len(returns) == 0: diff --git a/tools/autograd/gen_variable_type.py b/tools/autograd/gen_variable_type.py index d7c4dcc4dce7..989615ad14b2 100644 --- a/tools/autograd/gen_variable_type.py +++ b/tools/autograd/gen_variable_type.py @@ -24,7 +24,7 @@ # from __future__ import print_function from .utils import CodeTemplate, nested_dict, write, uninplace_api_name -from .gen_autograd import VIEW_FUNCTIONS, RETURNS_VIEWS_OF_INPUT, PURE_VIEW_FUNCTIONS +from .gen_autograd import VIEW_FUNCTIONS from .gen_autograd_functions import uses_single_grad # These functions we don't want to record for tracing, because we always want @@ -562,8 +562,6 @@ def emit_body(declaration): base_name = name[:-1] if inplace else name[:-4] if is_out_fn else name view_info = VIEW_FUNCTIONS.get(base_name, None) - if view_info is None and base_name in RETURNS_VIEWS_OF_INPUT: - view_info = "self" def is_differentiable(arg): if 'TensorOptions' in arg['type']: @@ -761,39 +759,59 @@ def emit_body(declaration): return '\n'.join(names) def wrap_output(call): - # Returns `wrapped_call` which is a drop-in replacement for `call` + # Returns a 2-tuple `(wrapped_call, extra_wrapping_stmts)`, where + # `wrapped_call` is to drop-in replace `call`, and + # `extra_wrapping_stmts` is a list of extra statements to run after + # `call`. if 'Tensor' not in declaration['return_type']: - return call + return call, [] elif view_info is not None: # See NOTE [ Autograd View Variables ] in variable.h for details. differentiable_output_vars = {r['name'] for r in differentiable_outputs} - - if not isinstance(view_info, str): - raise TypeError("The view info should be a string for {}, but it is: {}".format(base_name, view_info)) - - if len(differentiable_output_vars) == 0: - # no output is differentiable (.indices() for SparseTensors for example) - return 'as_non_differentiable_view({}, {})'.format(view_info, call) - elif len(differentiable_output_vars) == 1: - # Single differentiable output (Tensor or Tensor[]) - return_info = returns[0] - # We only support simple Tensor or a TensorList for functions that return views - if not return_info['dynamic_type'] in ['Tensor', 'TensorList']: - raise RuntimeError("{} that return differentiable views can only return Tensor or Tensor[]".format(base_name)) - # Only allow rebasing of the history if we return a single Tensor - # Or if the function is marked as a pure view - allow_rebase_history = 'true' - if return_info['dynamic_type'] == 'TensorList' and base_name not in PURE_VIEW_FUNCTIONS: - allow_rebase_history = 'false' - wrapped_call = ("as_differentiable_view(/* base */{}, /* output */ {}, " - "/* allow_rebase_history */ {})").format(view_info, call, allow_rebase_history) - return wrapped_call + tensor_output_vars = {r['name'] for r in returns if 'Tensor' in r['type']} + if not isinstance(view_info, dict): + if len(differentiable_output_vars) == len(tensor_output_vars): + # all outputs are differentiable + return 'as_view({}, {}, true)'.format(view_info, call), [] + elif len(differentiable_output_vars) == 0: + # no output is differentiable + return 'as_view({}, {}, false)'.format(view_info, call), [] + else: + # some of the outputs are differentiable + # need to expand to dict mode, i.e., one entry per output + base_name = view_info + view_info_dict = {} + for i, return_info in enumerate(returns): + if 'Tensor' in return_info['type']: + view_info_dict[i] = base_name else: - # This could be supported but we don't need it at the moment, so keeping things simple. - raise RuntimeError("Function that return multiple differentiable output " - "when at least one of them is view is not supported.") + view_info_dict = view_info + + def wrap_view_single(output_var, base_var): + fmt = '{output_var} = as_view({base_var}, {output_var}, {is_differentiable});' + if output_var in differentiable_output_vars: + # If `GradMode::is_enabled()` is False, this is a + # non-differentiable view. Gradients should not flow through. + is_differentiable = 'true' + else: + # This output is non-differentiable, so it is a + # non-differentiable view. Gradients should not flow through. + is_differentiable = 'false' + return fmt.format(output_var=output_var, base_var=base_var, + is_differentiable=is_differentiable) + + extra_wrapping_stmts = [] + for output_idx, return_info in enumerate(returns): + if 'Tensor' not in return_info['type']: + assert output_idx not in view_info_dict, 'Can not wrap non-Tensor output as a view' + continue + output_var = return_info['name'] + if output_idx in view_info_dict: + stmt = wrap_view_single(output_var, view_info_dict[output_idx]) + extra_wrapping_stmts.append(stmt) + return call, extra_wrapping_stmts else: - return 'std::move({})'.format(call) + return 'std::move({})'.format(call), [] def enforce_same_tensorimpl_and_storage(env, call): save_ptrs_stmts = [] @@ -820,6 +838,7 @@ def emit_body(declaration): def emit_call(env): combined = nested_dict(env, declaration) + extra_wrapping_stmts = [] if strategy == 'use_derived': # We only care about adding `at::AutoNonVariableTypeMode` guard for non-variable dispatch # (which corresponds to 'use_derived' strategy). The purpose of this guard is to make sure @@ -833,7 +852,7 @@ def emit_body(declaration): base_type_call = CALL_DISPATCH_VIA_METHOD.substitute( combined, unpacked_method_args=unpacked_method_args) if not modifies_arguments and not returns_void: - rhs_value = wrap_output('tmp') + rhs_value, extra_wrapping_stmts = wrap_output('tmp') call = DISPATCH_TO_NON_VAR_TYPE_WITH_RETURN_VALUES.substitute( base_type_call=base_type_call, return_values=tie_return_values(), @@ -846,6 +865,8 @@ def emit_body(declaration): if not modifies_arguments and not returns_void: call = '{} = {}'.format(tie_return_values(), call) call = call + ';' + for stmt in extra_wrapping_stmts: + call += '\n' + stmt call = enforce_same_tensorimpl_and_storage(env, call) return call diff --git a/torch/csrc/autograd/VariableTypeManual.cpp b/torch/csrc/autograd/VariableTypeManual.cpp index 57375b5b037f..b9aebfcc1f7f 100644 --- a/torch/csrc/autograd/VariableTypeManual.cpp +++ b/torch/csrc/autograd/VariableTypeManual.cpp @@ -255,7 +255,7 @@ Tensor detach(const Tensor & self) { } // - auto result = make_variable_non_differentiable_view(self, self, /*allow_tensor_metadata_change=*/false); + auto result = make_variable_view(self, self, /*is_differentiable=*/false, /*allow_tensor_metadata_change=*/false); namedinference::propagate_names(result, self); // if (jit::tracer::isTracing()) { diff --git a/torch/csrc/autograd/VariableTypeUtils.h b/torch/csrc/autograd/VariableTypeUtils.h index 8018716f25d0..4918a5aebe9c 100644 --- a/torch/csrc/autograd/VariableTypeUtils.h +++ b/torch/csrc/autograd/VariableTypeUtils.h @@ -41,20 +41,9 @@ namespace torch { namespace autograd { inline void check_inplace(const Tensor& tensor) { auto& var = static_cast(tensor); - if (var.requires_grad() && GradMode::is_enabled()) { - if (var.is_leaf()) { - AT_ERROR( - "a leaf Variable that requires grad is being used in an in-place operation."); - } else if (var.is_view()) { - // NB: is_view() ==> get_autograd_meta() - auto diff_view_meta = static_cast(impl::get_autograd_meta(var)); - auto grad_fn = impl::grad_fn_unsafe(var); - // NB: !var.is_leaf() ==> grad_fn != nullptr - TORCH_INTERNAL_ASSERT(grad_fn); - TORCH_CHECK(diff_view_meta->allow_rebase_history, - "The ", diff_view_meta->output_nr_, "th output of ", grad_fn->name(), - " is being modified inplace but this is not allowed as it would prevent correct gradient computation."); - } + if (var.requires_grad() && var.is_leaf() && GradMode::is_enabled()) { + AT_ERROR( + "a leaf Variable that requires grad has been used in an in-place operation."); } } @@ -108,44 +97,23 @@ template inline variable_list flatten_tensor_args(Args&&... ar } // See NOTE [ Autograd View Variables ] for details. -inline Tensor as_differentiable_view(const Tensor & base, Tensor tensor, bool allow_rebase_history) { +inline Tensor as_view(const Tensor & base, Tensor tensor, bool is_differentiable = true) { auto base_var = Variable(base); if (base_var.is_view()) { base_var = base_var.base(); } - return make_variable_differentiable_view(std::move(base_var), std::move(tensor), allow_rebase_history); + return make_variable_view(std::move(base_var), std::move(tensor), is_differentiable); } // See NOTE [ Autograd View Variables ] for details. -inline std::vector as_differentiable_view(const Tensor & base, std::vector tensors, - bool allow_rebase_history) { +inline std::vector as_view(const Tensor & base, std::vector tensors, + bool is_differentiable = true) { auto base_var = Variable(base); if (base_var.is_view()) { base_var = base_var.base(); } for(Tensor &tensor : tensors) { - tensor = make_variable_differentiable_view(base_var, std::move(tensor), allow_rebase_history); - } - return tensors; -} - -// See NOTE [ Autograd View Variables ] for details. -inline Tensor as_non_differentiable_view(const Tensor & base, Tensor tensor) { - auto base_var = Variable(base); - if (base_var.is_view()) { - base_var = base_var.base(); - } - return make_variable_non_differentiable_view(std::move(base_var), std::move(tensor)); -} - -// See NOTE [ Autograd View Variables ] for details. -inline std::vector as_non_differentiable_view(const Tensor & base, std::vector tensors) { - auto base_var = Variable(base); - if (base_var.is_view()) { - base_var = base_var.base(); - } - for(Tensor &tensor : tensors) { - tensor = make_variable_non_differentiable_view(base_var, std::move(tensor)); + tensor = make_variable_view(base_var, std::move(tensor), is_differentiable); } return tensors; } diff --git a/torch/csrc/autograd/variable.cpp b/torch/csrc/autograd/variable.cpp index 06805459bab4..d093db47141c 100644 --- a/torch/csrc/autograd/variable.cpp +++ b/torch/csrc/autograd/variable.cpp @@ -25,8 +25,8 @@ namespace torch { namespace autograd { -DifferentiableViewMeta::DifferentiableViewMeta(at::TensorImpl* self_impl, Variable base, bool allow_rebase_history) - : AutogradMeta(self_impl), allow_rebase_history(allow_rebase_history) { +DifferentiableViewMeta::DifferentiableViewMeta(at::TensorImpl* self_impl, Variable base) + : AutogradMeta(self_impl, false) { base_ = std::move(base); TORCH_CHECK(base_.defined(), "base is undefined"); if (base_.is_view()) { @@ -72,13 +72,12 @@ namespace impl { } void rebase_history(const Variable& self, Edge gradient_edge) { - TORCH_INTERNAL_ASSERT(gradient_edge.function != nullptr); + AT_ASSERT(gradient_edge.function != nullptr); if (self.is_view()) { // NB: is_view() ==> get_autograd_meta() auto diff_view_meta = static_cast(get_autograd_meta(self)); - TORCH_INTERNAL_ASSERT(diff_view_meta->allow_rebase_history); - TORCH_INTERNAL_ASSERT(gradient_edge.input_nr == 0); - TORCH_INTERNAL_ASSERT(gradient_edge.function); + AT_ASSERT(gradient_edge.input_nr == 0); + AT_ASSERT(gradient_edge.function); TORCH_CHECK( gradient_edge.function->num_inputs() == 1, "Functions which modify views in-place must return a single Variable"); diff --git a/torch/csrc/autograd/variable.h b/torch/csrc/autograd/variable.h index 3c07ec197407..a3fd96d57e6d 100644 --- a/torch/csrc/autograd/variable.h +++ b/torch/csrc/autograd/variable.h @@ -295,13 +295,6 @@ struct TORCH_API AutogradMeta : public c10::AutogradMetaInterface { /// + if an in-place op is done on view, in rebase_history() of view, which is /// called after every in-place op in VariableType.cpp, the grad_fn of base /// is updated. -/// + if a single autograd Node returns multiple differentiable views, if any -/// output is modified by an inplace operation, the autograd engine will make -/// an equivalent graph (corresponding to the view operations) without using -/// the user-provided grad_fn. If the provided grad_fn does more -/// than the backward of the view, then the DifferentiableViewMeta must be -/// created with allow_rebase_history=false to prevent the engine from -/// ignoring the provided grad_fn. /// /// /// Non-Differentiable Views @@ -320,9 +313,8 @@ struct TORCH_API AutogradMeta : public c10::AutogradMetaInterface { /// function are non-differentiable. /// These are called non-differentiable views as the gradients do not flow /// through the view relation. -/// -/// Relevant logic for both differentiable and non-differentiable views is implemented in -/// make_variable_(non_)differentiable_view below, and wrap_output of gen_variable_type.py. +/// Relevant logic for non-differentiable views is implemented in +/// make_variable_view below, and wrap_output of gen_variable_type.py. struct TORCH_API DifferentiableViewMeta : public AutogradMeta { /// The base `Variable` (never a view). Variable base_; @@ -332,15 +324,11 @@ struct TORCH_API DifferentiableViewMeta : public AutogradMeta { /// version_counter.current_version(). uint32_t attr_version; - /// Boolean flag that signifies if the history of this Tensor can be rebased - /// or if it is forbidden. - bool allow_rebase_history; - bool requires_grad() const override { return requires_grad_ || grad_fn_ || (is_view_ && base_.requires_grad()); } - DifferentiableViewMeta(at::TensorImpl* self_impl, Variable base, bool allow_rebase_history=true); + DifferentiableViewMeta(at::TensorImpl* self_impl, Variable base); ~DifferentiableViewMeta(); }; @@ -365,34 +353,28 @@ struct TORCH_API DifferentiableViewMeta : public AutogradMeta { /// prevent those changes from happening and is undesirable. // See NOTE [ Autograd View Variables ] for details. -// Differentiable view. Track history with DifferentiableViewMeta. -inline Variable make_variable_differentiable_view( - Variable base, - at::Tensor data, - bool allow_rebase_history) { - if (data.defined()) { - auto data_impl_copy = data.getIntrusivePtr()->shallow_copy_and_detach( - /*version_counter=*/0, - /*allow_tensor_metadata_change=*/true); - data_impl_copy->set_autograd_meta(std::make_unique( - data_impl_copy.get(), std::move(base), allow_rebase_history)); - return Variable(data_impl_copy); - } - return Variable(); -} - -// See NOTE [ Autograd View Variables ] for details. -// Non-differentiable view. Just share version counter. -inline Variable make_variable_non_differentiable_view( +inline Variable make_variable_view( Variable base, at::Tensor data, + bool is_differentiable = true, bool allow_tensor_metadata_change = true) { if (data.defined()) { - auto data_impl_copy = data.getIntrusivePtr()->shallow_copy_and_detach( - /*version_counter=*/impl::version_counter(base), - /*allow_tensor_metadata_change=*/allow_tensor_metadata_change); - data_impl_copy->set_autograd_meta(nullptr); - return Variable(data_impl_copy); + if (is_differentiable) { + /// Differentiable view. Track history with DifferentiableViewMeta. + auto data_impl_copy = data.getIntrusivePtr()->shallow_copy_and_detach( + /*version_counter=*/0, + /*allow_tensor_metadata_change=*/allow_tensor_metadata_change); + data_impl_copy->set_autograd_meta(std::make_unique( + data_impl_copy.get(), std::move(base))); + return Variable(data_impl_copy); + } else { + /// Non-differentiable view. Just share version counter. + auto data_impl_copy = data.getIntrusivePtr()->shallow_copy_and_detach( + /*version_counter=*/impl::version_counter(base), + /*allow_tensor_metadata_change=*/allow_tensor_metadata_change); + data_impl_copy->set_autograd_meta(nullptr); + return Variable(data_impl_copy); + } } return Variable(); }