From afaee00feca07c565f6b080e021b3422bfc1e8d4 Mon Sep 17 00:00:00 2001 From: Mikayla Gawarecki Date: Wed, 28 Sep 2022 17:38:29 +0000 Subject: [PATCH] Add python `nested_tensor` and `as_nested_tensor` constructors in `torch.nested` (#85593) Remove `torch.nested_tensor` which has erroneous behavior wrt gradients (could be either leaf or not leaf). Introduce `torch.nested.nested_tensor` and `torch.nested.as_nested_tensor` in the vein of `torch.tensor` and `torch.as_tensor`. Done in nested `__init__.py` for now but can move to pybind in future (when we want to load from numpy/nested lists ). Discussed offline with @cpuhrsch and pybind constructor (https://github.com/pytorch/pytorch/pull/85536) was more gnarly than expected, so we can move to that when we do need loading from numpy etc. Differential Revision: [D39806622](https://our.internmc.facebook.com/intern/diff/D39806622) Pull Request resolved: https://github.com/pytorch/pytorch/pull/85593 Approved by: https://github.com/drisspg, https://github.com/cpuhrsch --- aten/src/ATen/native/native_functions.yaml | 6 +- .../ATen/native/nested/NestedTensorMath.cpp | 2 +- docs/source/nested.rst | 16 +- test/cpp/api/nested.cpp | 2 +- .../check_forward_backward_compatibility.py | 2 + test/profiler/test_profiler.py | 2 +- test/test_autograd.py | 16 +- test/test_native_mha.py | 10 +- test/test_nestedtensor.py | 401 ++++++++++-------- test/test_nn.py | 10 +- test/test_torch.py | 8 +- test/test_transformers.py | 2 +- tools/autograd/derivatives.yaml | 2 +- torch/_subclasses/fake_tensor.py | 4 +- torch/csrc/api/include/torch/nested.h | 38 ++ torch/nested/__init__.py | 84 +++- torch/overrides.py | 1 - 17 files changed, 381 insertions(+), 225 deletions(-) diff --git a/aten/src/ATen/native/native_functions.yaml b/aten/src/ATen/native/native_functions.yaml index 3e141fcc3a64..70e78f71bd9a 100644 --- a/aten/src/ATen/native/native_functions.yaml +++ b/aten/src/ATen/native/native_functions.yaml @@ -12716,11 +12716,11 @@ variants: function python_module: nn -- func: nested_tensor(Tensor[] list, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor +- func: _nested_tensor_from_tensor_list(Tensor[] list, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor variants: function dispatch: - CompositeExplicitAutograd: nested_tensor - autogen: nested_tensor.out + CompositeExplicitAutograd: _nested_tensor_from_tensor_list + autogen: _nested_tensor_from_tensor_list.out - func: _fw_primal_copy(Tensor self, int level) -> Tensor variants: function diff --git a/aten/src/ATen/native/nested/NestedTensorMath.cpp b/aten/src/ATen/native/nested/NestedTensorMath.cpp index 84d62e91e4c9..6b8d711d4351 100644 --- a/aten/src/ATen/native/nested/NestedTensorMath.cpp +++ b/aten/src/ATen/native/nested/NestedTensorMath.cpp @@ -161,7 +161,7 @@ bool NestedTensor_nested_tensor_from_mask_left_aligned(const Tensor& t, const Te return sizes.equal(nums); } -Tensor nested_tensor( +Tensor _nested_tensor_from_tensor_list( TensorList list, c10::optional dtype, c10::optional layout, diff --git a/docs/source/nested.rst b/docs/source/nested.rst index 3fed648beac6..f4ef4316b306 100644 --- a/docs/source/nested.rst +++ b/docs/source/nested.rst @@ -23,7 +23,7 @@ Construction is straightforward and involves passing a list of Tensors to the co tensor([0, 1, 2]) >>> b tensor([3, 4, 5, 6, 7]) ->>> nt = torch.nested_tensor([a, b]) +>>> nt = torch.nested.nested_tensor([a, b]) >>> nt nested_tensor([ tensor([0, 1, 2]), @@ -32,7 +32,7 @@ nested_tensor([ Data type and device can be chosen via the usual keyword arguments. ->>> nt = torch.nested_tensor([a, b], dtype=torch.float32, device="cuda") +>>> nt = torch.nested.nested_tensor([a, b], dtype=torch.float32, device="cuda") >>> nt nested_tensor([ tensor([0., 1., 2.], device='cuda:0'), @@ -43,7 +43,7 @@ In order to form a valid NestedTensor the passed Tensors also all need to match >>> a = torch.randn(3, 50, 70) # image 1 >>> b = torch.randn(3, 128, 64) # image 2 ->>> nt = torch.nested_tensor([a, b], dtype=torch.float32) +>>> nt = torch.nested.nested_tensor([a, b], dtype=torch.float32) >>> nt.dim() 4 @@ -51,7 +51,7 @@ If one of the dimensions don't match, the constructor throws an error. >>> a = torch.randn(50, 128) # text 1 >>> b = torch.randn(3, 128, 64) # image 2 ->>> nt = torch.nested_tensor([a, b], dtype=torch.float32) +>>> nt = torch.nested.nested_tensor([a, b], dtype=torch.float32) Traceback (most recent call last): File "", line 1, in RuntimeError: All Tensors given to nested_tensor must have the same dimension. Found dimension 3 for Tensor at index 1 and dimension 2 for Tensor at index 0. @@ -73,7 +73,7 @@ Even though a NestedTensor does not support .size() (or .shape), it supports .si >>> a = torch.randn(50, 128) # text 1 >>> b = torch.randn(32, 128) # text 2 ->>> nt = torch.nested_tensor([a, b], dtype=torch.float32) +>>> nt = torch.nested.nested_tensor([a, b], dtype=torch.float32) >>> nt.size(0) 2 >>> nt.size(1) @@ -86,7 +86,7 @@ RuntimeError: Given dimension 1 is irregular and does not have a size. If all dimensions are regular, the NestedTensor is intended to be semantically indistinguishable from a regular torch.Tensor. >>> a = torch.randn(20, 128) # text 1 ->>> nt = torch.nested_tensor([a, a], dtype=torch.float32) +>>> nt = torch.nested.nested_tensor([a, a], dtype=torch.float32) >>> nt.size(0) 2 >>> nt.size(1) @@ -112,7 +112,7 @@ unbind allows you to retrieve a view of the constituents. >>> import torch >>> a = torch.randn(2, 3) >>> b = torch.randn(3, 4) ->>> nt = torch.nested_tensor([a, b], dtype=torch.float32) +>>> nt = torch.nested.nested_tensor([a, b], dtype=torch.float32) >>> nt nested_tensor([ tensor([[ 1.2286, -1.2343, -1.4842], @@ -149,4 +149,6 @@ The following functions are related to nested tensors: .. currentmodule:: torch.nested +.. autofunction:: nested_tensor +.. autofunction:: as_nested_tensor .. autofunction:: to_padded_tensor diff --git a/test/cpp/api/nested.cpp b/test/cpp/api/nested.cpp index 938e05e4d44c..3900fad5bce9 100644 --- a/test/cpp/api/nested.cpp +++ b/test/cpp/api/nested.cpp @@ -10,6 +10,6 @@ TEST(NestedTest, Nested) { auto a = torch::randn({2, 3}); auto b = torch::randn({4, 5}); - auto nt = torch::nested_tensor({a, b}); + auto nt = torch::nested::nested_tensor({a, b}); torch::nested::to_padded_tensor(nt, 0); } diff --git a/test/forward_backward_compatibility/check_forward_backward_compatibility.py b/test/forward_backward_compatibility/check_forward_backward_compatibility.py index 1978aee14ca1..0698ea784f95 100644 --- a/test/forward_backward_compatibility/check_forward_backward_compatibility.py +++ b/test/forward_backward_compatibility/check_forward_backward_compatibility.py @@ -284,6 +284,8 @@ ALLOW_LIST = [ ("c10d::allgather_", datetime.date(2022, 10, 1)), ("aten::to_padded_tensor", datetime.date(2022, 10, 1)), ("aten::nested_to_padded_tensor", datetime.date(2022, 10, 1)), + ("aten::nested_tensor", datetime.date(2022, 10, 15)), + ] ALLOW_LIST_COMPILED = [ diff --git a/test/profiler/test_profiler.py b/test/profiler/test_profiler.py index e3070b6a249e..6ce156fe2c0a 100644 --- a/test/profiler/test_profiler.py +++ b/test/profiler/test_profiler.py @@ -1248,7 +1248,7 @@ class TestProfiler(TestCase): a = torch.randn(4, 4) b = torch.randn(4, 4) c = torch.randn(4, 4) - inp = torch.nested_tensor([a, b]) + inp = torch.nested.nested_tensor([a, b]) with torch.profiler.profile(record_shapes=True) as prof: torch.nn.functional.linear(inp, c, None) for e in prof.events(): diff --git a/test/test_autograd.py b/test/test_autograd.py index 058f9e3538bb..bc575c6b6308 100644 --- a/test/test_autograd.py +++ b/test/test_autograd.py @@ -3569,12 +3569,12 @@ class TestAutograd(TestCase): assert out_shape == torch.Size([10, 5]) assert grad_shape == torch.Size([5, 10]) - out = torch.nested_tensor([ + out = torch.nested.as_nested_tensor([ torch.randn(10, 5, requires_grad=True), torch.randn(10, 5, requires_grad=True), torch.randn(10, 5, requires_grad=True)] ) - grad = torch.nested_tensor([torch.randn(5, 10, requires_grad=True), torch.randn(5, 10, requires_grad=True)]) + grad = torch.nested.as_nested_tensor([torch.randn(5, 10, requires_grad=True), torch.randn(5, 10, requires_grad=True)]) out_shape, grad_shape = _calculate_shape(out, grad, False) assert torch.equal(out_shape, torch.tensor([[10, 5], [10, 5], [10, 5]])) @@ -9178,12 +9178,12 @@ class TestAutogradMultipleDispatch(TestCase): # test registered AutogradNestedTensor formula a = torch.arange(6, dtype=torch.float, device=device).reshape(2, 3).requires_grad_(True) b = torch.arange(8, dtype=torch.float, device=device).reshape(2, 4).requires_grad_(True) - nt = torch.nested_tensor([a, b], dtype=torch.float, device=device) + nt = torch.nested.as_nested_tensor([a, b], dtype=torch.float, device=device) nt_out = torch._test_autograd_multiple_dispatch(nt) c = torch.randn(2, 3, device=device) d = torch.randn(2, 4, device=device) - nt_grad = torch.nested_tensor([c, d], dtype=torch.float, device=device) + nt_grad = torch.nested.nested_tensor([c, d], dtype=torch.float, device=device) nt_out.backward(nt_grad) # bogus gradient for AutogradNestedTensor is grad * grad @@ -9204,12 +9204,12 @@ class TestAutogradMultipleDispatch(TestCase): # test registered AutogradNestedTensor formula a = torch.arange(6, dtype=torch.float, device=device).reshape(2, 3).requires_grad_(True) b = torch.arange(8, dtype=torch.float, device=device).reshape(2, 4).requires_grad_(True) - nt = torch.nested_tensor([a, b], dtype=torch.float, device=device) + nt = torch.nested.as_nested_tensor([a, b], dtype=torch.float, device=device) nt_out = torch._test_autograd_multiple_dispatch(nt, True) c = torch.randn(2, 3, device=device) d = torch.randn(2, 4, device=device) - nt_grad = torch.nested_tensor([c, d], dtype=torch.float, device=device) + nt_grad = torch.nested.nested_tensor([c, d], dtype=torch.float, device=device) nt_out.backward(nt_grad) # bogus gradient for AutogradNestedTensor is grad * grad + grad @@ -9274,9 +9274,9 @@ class TestAutogradMultipleDispatch(TestCase): foo(inp).backward() # sum's input is saved for Nested Tensors - nt = torch.nested_tensor([torch.rand(2), torch.rand(2)], device=device).requires_grad_() + nt = torch.nested.nested_tensor([torch.rand(2), torch.rand(2)], device=device, requires_grad=True) with self.assertRaisesRegex(RuntimeError, "modified by an inplace operation"): - foo(nt).backward(torch.nested_tensor([torch.rand(1), torch.rand(1)], device=device)) + foo(nt).backward(torch.nested.nested_tensor([torch.rand(1), torch.rand(1)], device=device)) # Import test cases from below autograd/ here. These are found # implicitly by the loader, so Flake8 thinks they are unused, hence diff --git a/test/test_native_mha.py b/test/test_native_mha.py index 2d0843b5a4d3..41f56e8b8929 100644 --- a/test/test_native_mha.py +++ b/test/test_native_mha.py @@ -39,7 +39,7 @@ class TestMHADeviceType(TestCase): xs = list(torch.unbind(x)) if use_padding: xs[0] = xs[0][:-1] - x = torch.nested_tensor(xs, device=device, dtype=dtype) + x = torch.nested.nested_tensor(xs, device=device, dtype=dtype) qkv = torch.nn.Linear(embed_dim, 3 * embed_dim, device=device, dtype=dtype) # We have to use inference_mode here because q/k/v are @@ -199,15 +199,15 @@ class TestMHADeviceType(TestCase): qs = [x[:-1] for x in qs] else: qs[0] = qs[0][:-1] - q = torch.nested_tensor(qs, device=device, dtype=dtype) + q = torch.nested.nested_tensor(qs, device=device, dtype=dtype) if mode == "self": k = v = q elif mode == "encdec": - k = torch.nested_tensor(torch.unbind(k), device=device, dtype=dtype) + k = torch.nested.nested_tensor(torch.unbind(k), device=device, dtype=dtype) v = k else: - k = torch.nested_tensor(torch.unbind(k), device=device, dtype=dtype) - v = torch.nested_tensor(torch.unbind(v), device=device, dtype=dtype) + k = torch.nested.nested_tensor(torch.unbind(k), device=device, dtype=dtype) + v = torch.nested.nested_tensor(torch.unbind(v), device=device, dtype=dtype) ynpt, weight_npt = npt( q, k, v, key_padding_mask=mask if use_padding and not use_nt else None diff --git a/test/test_nestedtensor.py b/test/test_nestedtensor.py index 9de07c9b30e6..24671ca090d7 100644 --- a/test/test_nestedtensor.py +++ b/test/test_nestedtensor.py @@ -12,13 +12,12 @@ from torch.testing._internal.common_device_type import ( ) from torch.testing._internal.common_dtype import floating_types_and_half from torch.testing._internal.common_utils import TestCase, IS_FBCODE, run_tests, freeze_rng_state, parametrize, gradcheck -from torch import nested_tensor # Tests are ported from pytorch/nestedtensor. # This makes porting as_nested_tensor easier in the future. def _iter_constructors(): # yield as_nested_tensor - yield nested_tensor + yield torch.nested.nested_tensor # Helper function to generate a pair of random nested tensors # one is contiguous, the other is not, but they appear to have same entries @@ -33,10 +32,10 @@ def random_nt_noncontiguous_pair(ragged_sizes, device="cpu", dtype=torch.float16 ys = [] for x in xs: ys.append(x.transpose(-1, -2)) - nt_contiguous = torch.nested_tensor(ys) + nt_contiguous = torch.nested.nested_tensor(ys) # noncontiguous nested tensor n = len(ragged_sizes) - nt_noncontiguous = torch.nested_tensor(xs).transpose(-1, -2) + nt_noncontiguous = torch.nested.nested_tensor(xs).transpose(-1, -2) return nt_contiguous, nt_noncontiguous # Helper functions to pad a noncontiguous nested tensor @@ -67,18 +66,18 @@ class TestNestedTensor(TestCase): @torch.inference_mode() def _test_unbind_case(self, a, b): - nt = nested_tensor([a, b]) + nt = torch.nested.nested_tensor([a, b]) a1, b1 = nt.unbind() self.assertTrue(a is not a1) self.assertTrue(b is not b1) - nt = nested_tensor([a, b], dtype=a.dtype) + nt = torch.nested.nested_tensor([a, b], dtype=a.dtype) a1, b1 = nt.unbind(0) self.assertEqual(a, a1) self.assertEqual(b, b1) a = torch.randn((2, 3)).add_(1) - nt = nested_tensor([a]) + nt = torch.nested.nested_tensor([a]) self.assertEqual(a, nt.unbind(0)[0]) @torch.inference_mode() @@ -110,7 +109,7 @@ class TestNestedTensor(TestCase): def _test_fn(unbind_fn): a = torch.rand(3, 2) b = torch.rand(2, 3) - nt = nested_tensor([a, b]) + nt = torch.nested.nested_tensor([a, b]) self.assertRaises(RuntimeError, lambda: unbind_fn(nt, 1)) # Both of these tests are necessary, because we're using @@ -121,29 +120,29 @@ class TestNestedTensor(TestCase): @torch.inference_mode() def test_nested_tensor(self): - self.assertRaises(TypeError, lambda: nested_tensor([3.0])) - self.assertRaises(TypeError, lambda: nested_tensor(torch.tensor([3.0]))) - self.assertRaises(TypeError, lambda: nested_tensor(4.0)) + self.assertRaises(TypeError, lambda: torch.nested.nested_tensor([3.0])) + self.assertRaises(TypeError, lambda: torch.nested.nested_tensor(torch.tensor([3.0]))) + self.assertRaises(TypeError, lambda: torch.nested.nested_tensor(4.0)) @torch.inference_mode() def test_nested_tensor_matching_dim(self): self.assertRaisesRegex( RuntimeError, "Found dimension 1 for Tensor at index 1 and dimension 0 for Tensor at index 0.", - lambda: nested_tensor([torch.tensor(1.0), torch.tensor([])]), + lambda: torch.nested.nested_tensor([torch.tensor(1.0), torch.tensor([])]), ) self.assertRaisesRegex( RuntimeError, "Found dimension 1 for Tensor at index 2 and dimension 0 for Tensor at index 1.", - lambda: nested_tensor( + lambda: torch.nested.nested_tensor( [torch.tensor(1.0), torch.tensor(2.0), torch.tensor([])] ), ) @torch.inference_mode() def test_default_nested_tensor(self): - self.assertRaises(TypeError, lambda: nested_tensor()) - default_nested_tensor = nested_tensor([]) + self.assertRaises(TypeError, lambda: torch.nested.nested_tensor()) + default_nested_tensor = torch.nested.nested_tensor([]) default_tensor = torch.tensor([]) # self.assertEqual(default_nested_tensor.nested_dim(), 1) # self.assertEqual(default_nested_tensor.nested_size(), ()) @@ -203,6 +202,30 @@ class TestNestedTensor(TestCase): lambda: a1.size(), ) + def test_size_dim(self): + a = torch.nested.nested_tensor([]) + self.assertEqual(a.size(0), 0) + + a = torch.nested.nested_tensor([torch.tensor(1)]) + self.assertEqual(a.size(0), 1) + + a = torch.nested.nested_tensor([torch.tensor(1), torch.tensor(2)]) + self.assertEqual(a.size(0), 2) + + a = torch.nested.nested_tensor([torch.rand(1, 2), + torch.rand(1, 8)]) + self.assertEqual(a.size(0), 2) + self.assertEqual(a.size(1), 1) + self.assertRaisesRegex( + RuntimeError, "Given dimension 2 is irregular and does not have a size", lambda: a.size(2)) + + a = torch.nested.nested_tensor([torch.rand(3, 4), + torch.rand(5, 4)]) + self.assertEqual(a.size(0), 2) + self.assertRaisesRegex( + RuntimeError, "Given dimension 1 is irregular and does not have a size", lambda: a.size(1)) + self.assertEqual(a.size(2), 4) + @unittest.skipIf(IS_FBCODE, "stride is not virtual in fbcode.") @torch.inference_mode() def test_stride(self): @@ -218,7 +241,7 @@ class TestNestedTensor(TestCase): @torch.inference_mode() def test_is_contiguous(self): # Test empty case - nt_empty = torch.nested_tensor([]) + nt_empty = torch.nested.nested_tensor([]) assert nt_empty.is_contiguous() self.assertEqual(nt_empty, nt_empty.contiguous()) @@ -234,17 +257,17 @@ class TestNestedTensor(TestCase): @torch.inference_mode() def test_repr_string(self): - a = nested_tensor([]) + a = torch.nested.nested_tensor([]) expected = "nested_tensor([" "\n\n])" self.assertEqual(str(a), expected) self.assertEqual(repr(a), expected) - a = nested_tensor([torch.tensor(1.0)]) + a = torch.nested.nested_tensor([torch.tensor(1.0)]) expected = "nested_tensor([" "\n tensor(1.)" "\n])" self.assertEqual(str(a), expected) self.assertEqual(repr(a), expected) - a = nested_tensor([torch.tensor([[1, 2]]), torch.tensor([[4, 5]])]) + a = torch.nested.nested_tensor([torch.tensor([[1, 2]]), torch.tensor([[4, 5]])]) expected = ( "nested_tensor([" "\n tensor([[1, 2]])" "," "\n tensor([[4, 5]])" "\n])" ) @@ -255,19 +278,19 @@ class TestNestedTensor(TestCase): def test_activations(self): for func in (torch.nn.functional.relu, torch.nn.functional.relu_, torch.nn.functional.gelu, torch._C._nn.gelu_): t = torch.tensor([-1, 0, 1], dtype=torch.float) - nt = nested_tensor([t]) + nt = torch.nested.nested_tensor([t]) nested_result = func(nt) self.assertTrue(nested_result.is_nested) self.assertEqual(func(t), nested_result.unbind()[0]) def test_to_padded_tensor_on_empty_tensor(self): - nt = torch.nested_tensor([]) + nt = torch.nested.nested_tensor([]) empty = torch.nested.to_padded_tensor(nt, 4) self.assertEqual(empty, torch.tensor([])) def test_nested_namespace(self): - nt = torch.nested_tensor([torch.randn(2, 3), torch.randn(4, 5)]) + nt = torch.nested.nested_tensor([torch.randn(2, 3), torch.randn(4, 5)]) result = nt.to_padded_tensor(4) nested_namespace_result = torch.nested.to_padded_tensor(nt, 4) self.assertEqual(result, nested_namespace_result) @@ -284,7 +307,7 @@ class TestNestedTensorDeviceType(TestCase): for (min_dim, max_dim) in zip(min_dims, max_dims)]) t1 = torch.randn(tensor_dims, device=device, dtype=dtype) ts1.append(t1) - return torch.nested_tensor(ts1, device=device, dtype=dtype) + return torch.nested.nested_tensor(ts1, device=device, dtype=dtype) # Helper function to generate a pair of random nested tensors # the 2 nested tensors have same shapes @@ -297,14 +320,14 @@ class TestNestedTensorDeviceType(TestCase): t2 = torch.randn(tensor_dims, device=device, dtype=dtype) ts1.append(t1) ts2.append(t2) - return (torch.nested_tensor(ts1, device=device, dtype=dtype), - torch.nested_tensor(ts2, device=device, dtype=dtype)) + return (torch.nested.nested_tensor(ts1, device=device, dtype=dtype), + torch.nested.nested_tensor(ts2, device=device, dtype=dtype)) @dtypes(*floating_types_and_half()) def test_detach(self, device, dtype): a = torch.randn(2, 4, device=device, dtype=dtype, requires_grad=False) b = torch.randn(5, 4, device=device, dtype=dtype, requires_grad=False) - x = torch.nested_tensor([a, b]).requires_grad_() + x = torch.nested.nested_tensor([a, b], requires_grad=True) x_detach = x.detach() @@ -314,7 +337,7 @@ class TestNestedTensorDeviceType(TestCase): a = torch.randn(2, 4, device=device, dtype=dtype, requires_grad=True) b = torch.randn(5, 4, device=device, dtype=dtype, requires_grad=True) - x = torch.nested_tensor([a, b]) + x = torch.nested.as_nested_tensor([a, b]) y = x * 2 y = y.detach() @@ -345,7 +368,7 @@ class TestNestedTensorDeviceType(TestCase): t = torch.randn(4, 4, 4, device=device, dtype=dtype) ts = list(torch.unbind(t)) ts[0] = ts[0][:-1] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) padded = torch.nested.to_padded_tensor(nt, 0) nt_to = torch._nested_from_padded_and_nested_example(padded, nt) @@ -363,7 +386,7 @@ class TestNestedTensorDeviceType(TestCase): t0 = torch.randn(2, size, device=device, dtype=dtype, requires_grad=False) t1 = torch.randn(2, size, device=device, dtype=dtype, requires_grad=False) ts = [t0, t1, t0, t1] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) layer_norm = torch.nn.LayerNorm(size, device=device, dtype=dtype) nt_result = nt._nested_tensor_layer_norm( layer_norm.weight, layer_norm.bias, 1e-5 @@ -382,7 +405,7 @@ class TestNestedTensorDeviceType(TestCase): torch.randint(100, (L,), device=device, dtype=torch.int64) for L in torch.randint(5, 50, (8,)) ] - x = torch.nested_tensor(inputs, device=device, dtype=torch.int64) + x = torch.nested.nested_tensor(inputs, device=device, dtype=torch.int64) emb = torch.nn.Embedding(100, 8, device=device) y = emb(x) ys = y.unbind() @@ -394,7 +417,7 @@ class TestNestedTensorDeviceType(TestCase): t = torch.randn(4, 4, 4, device=device, dtype=dtype) ts = list(torch.unbind(t)) ts[0] = ts[0][:-1] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) for padding_value in (0, 1): padded = torch.nested.to_padded_tensor(nt, padding_value) @@ -414,7 +437,7 @@ class TestNestedTensorDeviceType(TestCase): output_size = (4, 6, 5) ts = list(torch.unbind(t)) ts[0] = ts[0][:-1] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) for padding_value in (0, 1): padded = torch.nested.to_padded_tensor(nt, padding_value, output_size=output_size) correct_output = torch.ones(output_size, device=device, dtype=dtype) * padding_value @@ -435,7 +458,7 @@ class TestNestedTensorDeviceType(TestCase): torch.randn(1240, device=device, dtype=dtype), torch.randn(2400, device=device, dtype=dtype), ] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) pad = 42 correct_output = [] for t in ts: @@ -453,7 +476,7 @@ class TestNestedTensorDeviceType(TestCase): torch.randn(24, 32, device=device, dtype=dtype), torch.randn(40, 53, device=device, dtype=dtype), ] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) pad = 42 correct_output = [] for t in ts: @@ -471,7 +494,7 @@ class TestNestedTensorDeviceType(TestCase): torch.randn(24, 32, 14, device=device, dtype=dtype), torch.randn(40, 53, 16, device=device, dtype=dtype), ] - nt = torch.nested_tensor(ts, device=device, dtype=dtype) + nt = torch.nested.nested_tensor(ts, device=device, dtype=dtype) pad = 42 correct_output = [] for t in ts: @@ -503,19 +526,19 @@ class TestNestedTensorDeviceType(TestCase): @skipMeta def test_device_checks(self, device): - nt = torch.nested_tensor([], device=device) + nt = torch.nested.nested_tensor([], device=device) is_cuda = 'cuda' in str(device) self.assertEqual(nt.is_cuda, is_cuda) @dtypes(torch.float, torch.float16, torch.double) def test_nested_tensor_indexing(self, device, dtype): # edge case: empty nested tensor - nt0 = torch.nested_tensor([]) + nt0 = torch.nested.nested_tensor([]) self.assertRaises(IndexError, lambda: nt0[0]) # normal case x0 = torch.randn((2, 5), device=device, dtype=dtype) x1 = torch.randn((3, 4), device=device, dtype=dtype) - nt = torch.nested_tensor([x0, x1]) + nt = torch.nested.nested_tensor([x0, x1]) # single index: only support integer in the batch dimension self.assertEqual(nt[0], x0) self.assertEqual(nt[-1], x1) @@ -539,16 +562,6 @@ class TestNestedTensorDeviceType(TestCase): answer = torch.tensor(200.0, device=device, dtype=dtype).expand(4) self.assertEqual(nt[1, 1, :], answer) - # Test that indexing works when requires_grad_(True) - # previously this was failing because the backward kernel for select.int uses .sizes() - nt = torch.nested_tensor([x0, x1]).requires_grad_(True) - self.assertEqual(nt[0], x0) - self.assertEqual(nt[-1], x1) - grad_x0 = torch.randn((2, 5), device=device, dtype=dtype) - nt[0].backward(grad_x0) - expected_grad = torch.nested_tensor([grad_x0, torch.zeros((3, 4), device=device, dtype=dtype)]) - self.assertEqual(nt.grad, expected_grad) - @dtypes(torch.float, torch.float16, torch.double) @torch.inference_mode() def test_nested_tensor_indexing_noncontiguous(self, device, dtype): @@ -563,7 +576,7 @@ class TestNestedTensorDeviceType(TestCase): @torch.inference_mode() def test_nested_tensor_add(self, device, dtype): (nt1, nt2) = self.random_nt_pair(device, dtype, 4, (4, 4)) - ref = torch.nested_tensor([t1 + t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) + ref = torch.nested.nested_tensor([t1 + t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) out = nt1 + nt2 self.assertEqual(ref, out) @@ -573,13 +586,13 @@ class TestNestedTensorDeviceType(TestCase): def test_nested_tensor_mul(self, device, dtype): # nested tensor * nested tensor (nt1, nt2) = self.random_nt_pair(device, dtype, 4, (4, 4)) - ref = torch.nested_tensor([t1 * t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) + ref = torch.nested.nested_tensor([t1 * t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) out = nt1 * nt2 self.assertEqual(ref, out) # nested tensor * scalar number = 10.0 scalar = torch.tensor(number).to(dtype).to(device) - ref = torch.nested_tensor([t * number for t in nt1.unbind()]) + ref = torch.nested.nested_tensor([t * number for t in nt1.unbind()]) out_number0 = nt1 * number out_number1 = number * nt1 out_scalar0 = nt1 * scalar @@ -606,7 +619,7 @@ class TestNestedTensorDeviceType(TestCase): @torch.inference_mode() def test_nested_tensor_add_in_place(self, device, dtype): (nt1, nt2) = self.random_nt_pair(device, dtype, 4, (4, 4)) - ref = torch.nested_tensor([t1 + t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) + ref = torch.nested.nested_tensor([t1 + t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) nt1 += nt2 self.assertEqual(ref, nt1) @@ -616,13 +629,13 @@ class TestNestedTensorDeviceType(TestCase): def test_nested_tensor_mul_in_place(self, device, dtype): # nested tensor * nested tensor (nt1, nt2) = self.random_nt_pair(device, dtype, 4, (4, 4)) - ref = torch.nested_tensor([t1 * t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) + ref = torch.nested.nested_tensor([t1 * t2 for (t1, t2) in zip(nt1.unbind(), nt2.unbind())]) nt1 *= nt2 self.assertEqual(ref, nt1) # nested tensor * scalar number = 10.0 scalar = torch.tensor(number).to(dtype).to(device) - ref = torch.nested_tensor([t * number for t in nt1.unbind()]) + ref = torch.nested.nested_tensor([t * number for t in nt1.unbind()]) out_number = nt1.clone() out_number *= number out_scalar = nt1.clone() @@ -661,7 +674,7 @@ class TestNestedTensorDeviceType(TestCase): [t.requires_grad_(True) for t in ub2] nt_sum = nt.sum(dim=dim, keepdim=keepdim) ub2_sum = [t.sum(-1, keepdim=keepdim) for t in ub2] - self.assertEqual(nt_sum, torch.nested_tensor(ub2_sum)) + self.assertEqual(nt_sum, torch.nested.nested_tensor(ub2_sum)) # test backward # generate gradient tensor that has the same size as the output @@ -669,11 +682,11 @@ class TestNestedTensorDeviceType(TestCase): gt2 = [] for i in range(ntensors): gt2.append(torch.randn(size[i].tolist(), device=device, dtype=dtype)) - gt = torch.nested_tensor(gt2).clone() + gt = torch.nested.nested_tensor(gt2).clone() nt_sum.backward(gt) for t2, g2 in zip(ub2_sum, gt2): t2.backward(g2) - self.assertEqual(nt.grad, torch.nested_tensor([t.grad for t in ub2])) + self.assertEqual(nt.grad, torch.nested.nested_tensor([t.grad for t in ub2])) return for ntensors, max_sizes in params: @@ -681,13 +694,13 @@ class TestNestedTensorDeviceType(TestCase): # Test error inputs with self.assertRaisesRegex(RuntimeError, "NestedTensor can only be reduced across the last"): - torch.nested_tensor([torch.tensor([3, 4, 5]), torch.tensor([1, 2])]).sum(0, keepdim=True) + torch.nested.nested_tensor([torch.tensor([3, 4, 5]), torch.tensor([1, 2])]).sum(0, keepdim=True) with self.assertRaisesRegex(RuntimeError, "NestedTensor only allows reduction of a single"): - torch.nested_tensor([torch.tensor([[3, 4, 5]]), torch.tensor([[1, 2]])]).sum([0, 1], keepdim=True) + torch.nested.nested_tensor([torch.tensor([[3, 4, 5]]), torch.tensor([[1, 2]])]).sum([0, 1], keepdim=True) with self.assertRaisesRegex(RuntimeError, "NestedTensor always requires keepdim=True for now."): - torch.nested_tensor([torch.tensor([3, 4, 5]), torch.tensor([1, 2])]).sum(-1) + torch.nested.nested_tensor([torch.tensor([3, 4, 5]), torch.tensor([1, 2])]).sum(-1) @dtypes(torch.float, torch.float16) @skipMeta @@ -712,7 +725,7 @@ class TestNestedTensorDeviceType(TestCase): @dtypes(torch.float, torch.double) def test_dropout(self, device, dtype): # edge case: empty nested tensor - nt0 = torch.nested_tensor([]) + nt0 = torch.nested.nested_tensor([]) y = torch.nn.functional.dropout(nt0, 0.5) self.assertEqual(nt0, y) # normal nested tensor @@ -802,11 +815,11 @@ class TestNestedTensorDeviceType(TestCase): expect = torch.nn.functional.softmax(pt, 1).nan_to_num_(0.0) self.assertEqual(torch.nested.to_padded_tensor(y0, 0.0), expect) # edge case: empty nested tensor - nt0 = torch.nested_tensor([]) + nt0 = torch.nested.nested_tensor([]) y = torch.nn.functional.softmax(nt0, 1) self.assertEqual(nt0, y) # edge case: nesting scalars - nt1 = torch.nested_tensor([torch.tensor(0.0), torch.tensor(1.0)]) + nt1 = torch.nested.nested_tensor([torch.tensor(0.0), torch.tensor(1.0)]) self.assertRaises(RuntimeError, lambda: torch.nn.functional.softmax(nt1, 0)) self.assertRaises(IndexError, lambda: torch.nn.functional.softmax(nt1, 1)) @@ -822,7 +835,7 @@ class TestNestedTensorDeviceType(TestCase): @dtypes(torch.float, torch.double) def test_bmm(self, device, dtype): # error case: one is nested but the other is not - nt = torch.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) + nt = torch.nested.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) t = torch.randn(4, device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, @@ -835,9 +848,9 @@ class TestNestedTensorDeviceType(TestCase): lambda: t.bmm(nt) ) # error case: not 3D tensors - nt0 = torch.nested_tensor([], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) - nt2 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) + nt2 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, "batch1 must be a 3D tensor", @@ -879,11 +892,11 @@ class TestNestedTensorDeviceType(TestCase): lambda: nt2.bmm(nt1) ) # error case: incompatible batch size - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((4, 6)), - torch.randn((4, 5)), - torch.randn((4, 7))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((4, 6)), + torch.randn((4, 5)), + torch.randn((4, 7))], + device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, "Expected size for the 1st dimension of batch2 tensor to be: 2 but got: 3.", @@ -895,15 +908,15 @@ class TestNestedTensorDeviceType(TestCase): lambda: nt1.bmm(nt0) ) # error case: underlying matrices cannot be multiplied - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, r"0-th nested matrices in batch cannot be multiplied \(2x4 and 2x4\)", lambda: nt0.bmm(nt0) ) # normal nested tensor - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 7))], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((4, 6)), torch.randn((7, 5))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 7))], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((4, 6)), torch.randn((7, 5))], device=device, dtype=dtype) actual = torch.nested.to_padded_tensor(nt0.bmm(nt1), 0.0) expect = torch.nested.to_padded_tensor(nt0, 0.0).bmm(torch.nested.to_padded_tensor(nt1, 0.0)) self.assertEqual(actual, expect) @@ -921,7 +934,7 @@ class TestNestedTensorDeviceType(TestCase): @dtypes(torch.float, torch.double) def test_matmul(self, device, dtype): # error case: one is nested but the other is not - nt = torch.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) + nt = torch.nested.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) t = torch.randn(4, device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, @@ -934,9 +947,9 @@ class TestNestedTensorDeviceType(TestCase): lambda: torch.matmul(t, nt) ) # error case: not 3+D tensors - nt0 = torch.nested_tensor([], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) - nt2 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn(2), torch.randn(3)], device=device, dtype=dtype) + nt2 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, r"matmul: For nested tensors, only inputs with >= 3 dims are currently supported. 1st input has rank: [0-9]+", @@ -978,11 +991,11 @@ class TestNestedTensorDeviceType(TestCase): lambda: torch.matmul(nt2, nt1) ) # error case: incompatible batch size - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((4, 6)), - torch.randn((4, 5)), - torch.randn((4, 7))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((4, 6)), + torch.randn((4, 5)), + torch.randn((4, 7))], + device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, r"matmul: Expected size for the 1st dimension of 2nd input tensor to be: [0-9]+ but got: [0-9]+.", @@ -994,59 +1007,59 @@ class TestNestedTensorDeviceType(TestCase): lambda: torch.matmul(nt1, nt0) ) # error case: incompatible (wrong) batch sizes that shouldn't even broadcast? - nt0 = torch.nested_tensor([torch.randn((2, 2, 4)), - torch.randn((2, 3, 4))], - device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((3, 4, 6)), - torch.randn((3, 4, 5))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 2, 4)), + torch.randn((2, 3, 4))], + device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((3, 4, 6)), + torch.randn((3, 4, 5))], + device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, "matmul(): For nested tensors, batch dimensions must have the same sizes,", lambda: torch.matmul(nt0, nt1) ) # error case: incompatible batch sizes that should technically broadcast - nt0 = torch.nested_tensor([torch.randn((2, 2, 4)), - torch.randn((1, 3, 4))], - device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((1, 4, 6)), - torch.randn((3, 4, 5))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 2, 4)), + torch.randn((1, 3, 4))], + device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((1, 4, 6)), + torch.randn((3, 4, 5))], + device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, "matmul(): For nested tensors, batch dimensions must have the same sizes,", lambda: torch.matmul(nt0, nt1) ) # error case: underlying matrices cannot be multiplied - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 4))], device=device, dtype=dtype) self.assertRaisesRegex( RuntimeError, "matmul(): Nested tensors cannot be matrix multiplied", lambda: torch.matmul(nt0, nt0) ) # normal nested tensor: 3D - nt0 = torch.nested_tensor([torch.randn((2, 4)), torch.randn((3, 7))], device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((4, 6)), torch.randn((7, 5))], device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((2, 4)), torch.randn((3, 7))], device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((4, 6)), torch.randn((7, 5))], device=device, dtype=dtype) actual = torch.nested.to_padded_tensor(torch.matmul(nt0, nt1), 0.0) expect = torch.matmul(torch.nested.to_padded_tensor(nt0, 0.0), torch.nested.to_padded_tensor(nt1, 0.0)) self.assertEqual(actual, expect) # normal nested tensor: 4D (with testing for batch_size=1) - nt0 = torch.nested_tensor([torch.randn((1, 2, 4)), - torch.randn((8, 3, 7))], - device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((1, 4, 6)), - torch.randn((8, 7, 5))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((1, 2, 4)), + torch.randn((8, 3, 7))], + device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((1, 4, 6)), + torch.randn((8, 7, 5))], + device=device, dtype=dtype) actual = torch.nested.to_padded_tensor(torch.matmul(nt0, nt1), 0.0) expect = torch.matmul(torch.nested.to_padded_tensor(nt0, 0.0), torch.nested.to_padded_tensor(nt1, 0.0)) self.assertEqual(actual, expect) # normal nested tensor: 5D - nt0 = torch.nested_tensor([torch.randn((8, 9, 2, 4)), - torch.randn((8, 9, 3, 7))], - device=device, dtype=dtype) - nt1 = torch.nested_tensor([torch.randn((8, 9, 4, 6)), - torch.randn((8, 9, 7, 5))], - device=device, dtype=dtype) + nt0 = torch.nested.nested_tensor([torch.randn((8, 9, 2, 4)), + torch.randn((8, 9, 3, 7))], + device=device, dtype=dtype) + nt1 = torch.nested.nested_tensor([torch.randn((8, 9, 4, 6)), + torch.randn((8, 9, 7, 5))], + device=device, dtype=dtype) actual = torch.nested.to_padded_tensor(torch.matmul(nt0, nt1), 0.0) expect = torch.matmul(torch.nested.to_padded_tensor(nt0, 0.0), torch.nested.to_padded_tensor(nt1, 0.0)) self.assertEqual(actual, expect) @@ -1065,7 +1078,7 @@ class TestNestedTensorDeviceType(TestCase): a = torch.randn(1, 2, device=device, dtype=dtype) b = torch.randn(2, 2, device=device, dtype=dtype) c = torch.randn(3, 2, device=device, dtype=dtype) - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.nested_tensor([a, b, c]) weight = torch.randn(2, 2, device=device, dtype=dtype) bias = torch.randn(2, device=device, dtype=dtype) @@ -1074,8 +1087,8 @@ class TestNestedTensorDeviceType(TestCase): # invalid nested tensor dimension msg = r'Linear requires nested_tensor.dim == 3 and dense_matrix.dim == 2. Nested tensor dim: 2. Dense tensor dim: 2' - nt1 = torch.nested_tensor([torch.randn(1, device=device, dtype=dtype), - torch.randn(2, device=device, dtype=dtype)]) + nt1 = torch.nested.nested_tensor([torch.randn(1, device=device, dtype=dtype), + torch.randn(2, device=device, dtype=dtype)]) with self.assertRaisesRegex(RuntimeError, msg): torch.functional.F.linear(nt1, weight, bias) @@ -1087,8 +1100,8 @@ class TestNestedTensorDeviceType(TestCase): # inconsistent last dim of nested tensor msg = r"Expected all tensors in nested tensor to have the same trailing dimension, instead last dimension equals:" - nt2 = torch.nested_tensor([torch.randn(1, 2, device=device, dtype=dtype), - torch.randn(2, 3, device=device, dtype=dtype)]) + nt2 = torch.nested.nested_tensor([torch.randn(1, 2, device=device, dtype=dtype), + torch.randn(2, 3, device=device, dtype=dtype)]) with self.assertRaisesRegex(RuntimeError, msg): torch.functional.F.linear(nt2, weight, bias) @@ -1172,7 +1185,7 @@ class TestNestedTensorDeviceType(TestCase): lambda: nt.view(()) ) # error case: empty nested tensor - nt_empty = torch.nested_tensor([]) + nt_empty = torch.nested.nested_tensor([]) self.assertRaisesRegex( RuntimeError, "empty nested tensor cannot be reshaped", @@ -1192,7 +1205,7 @@ class TestNestedTensorDeviceType(TestCase): # normal case x0 = torch.randn((2, 20), device=device, dtype=dtype) x1 = torch.randn((3, 20), device=device, dtype=dtype) - nt = torch.nested_tensor([x0, x1]) + nt = torch.nested.nested_tensor([x0, x1]) pt = torch.nested.to_padded_tensor(nt, 0.0) self.assertRaisesRegex( RuntimeError, @@ -1214,7 +1227,7 @@ class TestNestedTensorDeviceType(TestCase): @dtypes(torch.float, torch.float16, torch.double) def test_view_inference_mode_interaction(self, device, dtype): # Construct in default mode and view while in inference mode - nt = torch.nested_tensor([torch.randn((2, 20)), torch.randn((3, 20))], device=device, dtype=dtype) + nt = torch.nested.nested_tensor([torch.randn((2, 20)), torch.randn((3, 20))], device=device, dtype=dtype) with torch.inference_mode(): ntT = nt.view(2, -1, 4, 5) ptT_from_ntT = noncontiguous_to_padded_tensor(ntT) @@ -1223,7 +1236,7 @@ class TestNestedTensorDeviceType(TestCase): self.assertEqual(ptT, ptT_from_ntT) # Construct and view while in inference mode with torch.inference_mode(): - nt = torch.nested_tensor([torch.randn((2, 20)), torch.randn((3, 20))], device=device, dtype=dtype) + nt = torch.nested.nested_tensor([torch.randn((2, 20)), torch.randn((3, 20))], device=device, dtype=dtype) ntT = nt.view(2, -1, 4, 5) ptT_from_ntT = noncontiguous_to_padded_tensor(ntT) pt = torch.nested.to_padded_tensor(nt, 0.0) @@ -1240,7 +1253,7 @@ class TestNestedTensorDeviceType(TestCase): lambda: nt.reshape(()) ) # error case: empty nested tensor - nt_empty = torch.nested_tensor([]) + nt_empty = torch.nested.nested_tensor([]) self.assertRaisesRegex( RuntimeError, "empty nested tensor cannot be reshaped", @@ -1260,7 +1273,7 @@ class TestNestedTensorDeviceType(TestCase): # normal case x0 = torch.randn((2, 20), device=device, dtype=dtype) x1 = torch.randn((3, 20), device=device, dtype=dtype) - nt = torch.nested_tensor([x0, x1]) + nt = torch.nested.nested_tensor([x0, x1]) # (2, (2, 3), 20) pt = torch.nested.to_padded_tensor(nt, 0.0) self.assertRaisesRegex( RuntimeError, @@ -1288,17 +1301,17 @@ class TestNestedTensorDeviceType(TestCase): E = 10 if input_dim == 3: # Shape: (N, L, E); ragged L - query = torch.nested_tensor([rand_tensor(2, E), rand_tensor(3, E), rand_tensor(4, E)]) + query = torch.nested.nested_tensor([rand_tensor(2, E), rand_tensor(3, E), rand_tensor(4, E)]) # Shape: (N, S, E); ragged S - key = torch.nested_tensor([rand_tensor(3, E), rand_tensor(4, E), rand_tensor(5, E)]) - value = torch.nested_tensor([rand_tensor(3, E), rand_tensor(4, E), rand_tensor(5, E)]) + key = torch.nested.nested_tensor([rand_tensor(3, E), rand_tensor(4, E), rand_tensor(5, E)]) + value = torch.nested.nested_tensor([rand_tensor(3, E), rand_tensor(4, E), rand_tensor(5, E)]) elif input_dim == 4: # Shape: (N, N', L, E); ragged N' and L - query = torch.nested_tensor([rand_tensor(2, 2, E), rand_tensor(3, 3, E), rand_tensor(4, 4, E)]) + query = torch.nested.nested_tensor([rand_tensor(2, 2, E), rand_tensor(3, 3, E), rand_tensor(4, 4, E)]) # Shape: (N, N', S, E); ragged N' and S - key = torch.nested_tensor([rand_tensor(2, 3, E), rand_tensor(3, 4, E), rand_tensor(4, 5, E)]) - value = torch.nested_tensor([rand_tensor(2, 3, E), rand_tensor(3, 4, E), rand_tensor(4, 5, E)]) + key = torch.nested.nested_tensor([rand_tensor(2, 3, E), rand_tensor(3, 4, E), rand_tensor(4, 5, E)]) + value = torch.nested.nested_tensor([rand_tensor(2, 3, E), rand_tensor(3, 4, E), rand_tensor(4, 5, E)]) else: self.fail(f"Invalid input_dim {input_dim} encountered in SDP test") @@ -1306,7 +1319,7 @@ class TestNestedTensorDeviceType(TestCase): return torch.randint(0, 2, size=size, dtype=torch.bool, device=device) # Shape: (N, L, S); ragged L and S matching above - attn_mask = torch.nested_tensor([rand_mask((2, 3)), rand_mask((3, 4)), rand_mask((4, 5))]) + attn_mask = torch.nested.nested_tensor([rand_mask((2, 3)), rand_mask((3, 4)), rand_mask((4, 5))]) dropout_p = 0.0 # no dropout for reproducibility need_attn_weights: bool = True @@ -1323,8 +1336,8 @@ class TestNestedTensorDeviceType(TestCase): need_attn_weights=need_attn_weights) expected_outputs.append(output.squeeze(0)) expected_attn_weights.append(attn_weights.squeeze(0)) - expected_output_nested = torch.nested_tensor(expected_outputs) - expected_attn_weight_nested = torch.nested_tensor(expected_attn_weights) + expected_output_nested = torch.nested.nested_tensor(expected_outputs) + expected_attn_weight_nested = torch.nested.nested_tensor(expected_attn_weights) self.assertEqual(actual[0], expected_output_nested) self.assertEqual(actual[1], expected_attn_weight_nested) @@ -1343,15 +1356,55 @@ class TestNestedTensorAutograd(TestCase): # Note [Gradcheck args check_batched_grad=False] the common_utils testing version of gradcheck # includes the default parameters used for testing ops with gradcheck. However nested tensor # does not support the stack op therefore we turn it off for these tests + def _create_leaf_nested_tensor_from_list(self, requires_grad=False): + return torch.nested.nested_tensor([torch.randn(1, 2), + torch.randn(7, 8)], requires_grad=requires_grad) + def _create_nested_tensor_from_list(self, requires_grad=False): - return torch.nested_tensor([torch.randn(1, 2, requires_grad=requires_grad), - torch.randn(7, 8, requires_grad=requires_grad)]) + return torch.nested.as_nested_tensor([torch.randn(1, 2, requires_grad=requires_grad), + torch.randn(7, 8, requires_grad=requires_grad)]) + def _create_nested_tensor_from_mask(self, requires_grad=False): data = torch.randn(2, 3, 4, requires_grad=requires_grad) mask = torch.ones_like(data[:, :, 0]).bool() return torch._nested_tensor_from_mask(data, mask) + def test_as_nested_tensor_propagates_gradients(self): + a = torch.arange(3, dtype=torch.float) + b = torch.arange(5, dtype=torch.float) + nt = torch.nested.as_nested_tensor([a, b]) + # tensors with requires_grad=False are leaves + self.assertTrue(nt.is_leaf) + self.assertTrue(not nt.requires_grad) + + a = torch.arange(3, dtype=torch.float, requires_grad=True) + b = torch.arange(5, dtype=torch.float, requires_grad=True) + nt2 = torch.nested.as_nested_tensor([a, b]) + fake_grad = torch.nested.nested_tensor([torch.ones_like(a), torch.zeros_like(b)]) + nt2.backward(fake_grad) + self.assertEqual(a.grad, fake_grad[0]) + self.assertEqual(b.grad, fake_grad[1]) + + def test_nested_tensor_generates_leaf(self): + a = torch.arange(3, dtype=torch.float, requires_grad=True) + b = torch.arange(5, dtype=torch.float, requires_grad=True) + + nt = torch.nested.nested_tensor([a, b], requires_grad=False) + self.assertTrue(nt.is_leaf) + self.assertTrue(not nt.requires_grad) + + nt2 = torch.nested.nested_tensor([a, b], requires_grad=True) + self.assertTrue(nt2.is_leaf) + self.assertTrue(nt2.requires_grad) + + fake_grad = torch.nested.nested_tensor([torch.ones_like(a), torch.zeros_like(b)]) + nt2.backward(fake_grad) + self.assertEqual(nt2.grad, fake_grad) + self.assertEqual(a.grad, None) + self.assertEqual(b.grad, None) + + def test_set_requires_grad_from_list(self): nt = self._create_nested_tensor_from_list() nt.requires_grad_() @@ -1381,14 +1434,13 @@ class TestNestedTensorAutograd(TestCase): # Test Factory Functions def test_nested_tensor_to_padded_tensor(self): for padding_val in [0, 1]: - nt = torch.nested_tensor([torch.randn(1, 2), torch.randn(7, 8)]) - nt.requires_grad_() + nt = self._create_leaf_nested_tensor_from_list(True) out = torch.nested.to_padded_tensor(nt, padding_val) grad_output = torch.ones(out.shape) out.backward(grad_output) - self.assertEqual(nt.grad, torch.nested_tensor([torch.ones(1, 2), torch.ones(7, 8)])) + self.assertEqual(nt.grad, torch.nested.nested_tensor([torch.ones(1, 2), torch.ones(7, 8)])) def test_nested_tensor_from_mask_and_to_padded(self): N, L, D = 2, 4, 4 @@ -1442,41 +1494,17 @@ class TestNestedTensorAutograd(TestCase): c = torch.randn(10, 2, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c): - c = torch.nested_tensor([a, b, c]) + c = torch.nested.as_nested_tensor([a, b, c]) # This implictily tests to_padded_tensor grads return torch.nested.to_padded_tensor(c, 0) data = (a, b, c) assert gradcheck(grad_test_func, inputs=data, check_batched_grad=False) - def test_size_dim(self): - a = torch.nested_tensor([]) - self.assertEqual(a.size(0), 0) - - a = torch.nested_tensor([torch.tensor(1)]) - self.assertEqual(a.size(0), 1) - - a = torch.nested_tensor([torch.tensor(1), torch.tensor(2)]) - self.assertEqual(a.size(0), 2) - - a = torch.nested_tensor([torch.rand(1, 2), - torch.rand(1, 8)]) - self.assertEqual(a.size(0), 2) - self.assertEqual(a.size(1), 1) - self.assertRaisesRegex( - RuntimeError, "Given dimension 2 is irregular and does not have a size", lambda: a.size(2)) - - a = torch.nested_tensor([torch.rand(3, 4), - torch.rand(5, 4)]) - self.assertEqual(a.size(0), 2) - self.assertRaisesRegex( - RuntimeError, "Given dimension 1 is irregular and does not have a size", lambda: a.size(1)) - self.assertEqual(a.size(2), 4) - def test_dropout_backward(self): - nt = torch.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]).requires_grad_(True) + nt = torch.nested.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))], requires_grad=True) p = 0.2 y = torch.nn.functional.dropout(nt, p) - y.backward(nt.clone()) + y.backward(nt.clone().detach()) self.assertEqual(nt.grad, y) def test_nested_tensor_bmm_gradcheck(self): @@ -1486,8 +1514,8 @@ class TestNestedTensorAutograd(TestCase): d = torch.randn(6, 5, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c, d): - nt0 = torch.nested_tensor([a, b]) - nt1 = torch.nested_tensor([c, d]) + nt0 = torch.nested.as_nested_tensor([a, b]) + nt1 = torch.nested.as_nested_tensor([c, d]) result = nt0.bmm(nt1) return torch.nested.to_padded_tensor(result, 0.0) @@ -1495,8 +1523,8 @@ class TestNestedTensorAutograd(TestCase): assert torch.autograd.gradcheck(grad_test_func, inputs=data) def test_nested_tensor_bmm_backward(self): - nt0 = torch.nested_tensor([torch.randn((2, 6)), torch.randn((3, 6))]).requires_grad_(True) - nt1 = torch.nested_tensor([torch.randn((6, 4)), torch.randn((6, 5))]).requires_grad_(True) + nt0 = torch.nested.nested_tensor([torch.randn((2, 6)), torch.randn((3, 6))], requires_grad=True) + nt1 = torch.nested.nested_tensor([torch.randn((6, 4)), torch.randn((6, 5))], requires_grad=True) with torch.no_grad(): pt0 = torch.nested.to_padded_tensor(nt0, 0.0).requires_grad_(True) pt1 = torch.nested.to_padded_tensor(nt1, 0.0).requires_grad_(True) @@ -1516,8 +1544,8 @@ class TestNestedTensorAutograd(TestCase): d = torch.randn(6, 5, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c, d): - nt0 = torch.nested_tensor([a, b]) - nt1 = torch.nested_tensor([c, d]) + nt0 = torch.nested.as_nested_tensor([a, b]) + nt1 = torch.nested.as_nested_tensor([c, d]) result = torch.matmul(nt0, nt1) return torch.nested.to_padded_tensor(result, 0.0) @@ -1525,8 +1553,8 @@ class TestNestedTensorAutograd(TestCase): assert torch.autograd.gradcheck(grad_test_func, inputs=data) def test_nested_tensor_matmul_backward(self): - nt0 = torch.nested_tensor([torch.randn((7, 2, 6)), torch.randn((7, 3, 6))]).requires_grad_(True) - nt1 = torch.nested_tensor([torch.randn((7, 6, 4)), torch.randn((7, 6, 5))]).requires_grad_(True) + nt0 = torch.nested.nested_tensor([torch.randn((7, 2, 6)), torch.randn((7, 3, 6))], requires_grad=True) + nt1 = torch.nested.nested_tensor([torch.randn((7, 6, 4)), torch.randn((7, 6, 5))], requires_grad=True) with torch.no_grad(): pt0 = torch.nested.to_padded_tensor(nt0, 0.0).requires_grad_(True) pt1 = torch.nested.to_padded_tensor(nt1, 0.0).requires_grad_(True) @@ -1544,7 +1572,7 @@ class TestNestedTensorAutograd(TestCase): b = torch.randn(3, 4, requires_grad=True) def grad_test_func(a, b): - nt = torch.nested_tensor([a, b]) + nt = torch.nested.as_nested_tensor([a, b]) result = nt.transpose(-2, -1).transpose(-2, -1) return torch.nested.to_padded_tensor(result, 0.0) @@ -1552,7 +1580,7 @@ class TestNestedTensorAutograd(TestCase): assert torch.autograd.gradcheck(grad_test_func, inputs=data, eps=1e-3) def test_nested_tensor_transpose_backward(self): - nt = torch.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]).requires_grad_(True) + nt = torch.nested.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))], requires_grad=True) with torch.no_grad(): pt = torch.nested.to_padded_tensor(nt, 0.0).requires_grad_(True) @@ -1568,7 +1596,7 @@ class TestNestedTensorAutograd(TestCase): b = torch.randn(3, 6, requires_grad=True) def grad_test_func(a, b): - nt = torch.nested_tensor([a, b]) + nt = torch.nested.as_nested_tensor([a, b]) result = nt.reshape(2, -1, 2, 3) return torch.nested.to_padded_tensor(result, 0.0) @@ -1576,7 +1604,7 @@ class TestNestedTensorAutograd(TestCase): assert torch.autograd.gradcheck(grad_test_func, inputs=data, eps=1e-3) def test_nested_tensor_reshape_backward(self): - nt = torch.nested_tensor([torch.randn((2, 6)), torch.randn((3, 6))]).requires_grad_(True) + nt = torch.nested.nested_tensor([torch.randn((2, 6)), torch.randn((3, 6))], requires_grad=True) with torch.no_grad(): pt = torch.nested.to_padded_tensor(nt, 0.0).requires_grad_(True) @@ -1597,7 +1625,7 @@ class TestNestedTensorAutograd(TestCase): bias = torch.randn(2, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c, weight, bias=None): - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) # This implicitly tests to_padded_tensor grads d = torch.functional.F.linear(nt, weight, bias) return torch.nested.to_padded_tensor(d, 0) @@ -1614,7 +1642,7 @@ class TestNestedTensorAutograd(TestCase): c = torch.randn(3, 2, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c, dim): - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) # This implicitly tests to_padded_tensor grads d = torch.functional.F.softmax(nt, dim=dim) return torch.nested.to_padded_tensor(d, 0) @@ -1630,7 +1658,7 @@ class TestNestedTensorAutograd(TestCase): weight = torch.randn(2, 2, requires_grad=True) bias = torch.randn(2, requires_grad=True) - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) out = torch.functional.F.linear(nt, weight, bias) @@ -1649,7 +1677,7 @@ class TestNestedTensorAutograd(TestCase): c = torch.randn(3, 2, 4, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c): - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) buffer = nt.values() return buffer.sum() @@ -1662,7 +1690,7 @@ class TestNestedTensorAutograd(TestCase): c = torch.randn(1, 1, 2, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c): - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) buffer = nt.values() buffer = buffer * 2 return buffer.exp() @@ -1676,7 +1704,7 @@ class TestNestedTensorAutograd(TestCase): c = torch.randn(3, 2, 4, requires_grad=True, dtype=torch.float64) def grad_test_func(a, b, c): - nt = torch.nested_tensor([a, b, c]) + nt = torch.nested.as_nested_tensor([a, b, c]) ntT = nt.transpose(-1, -2) unbound = ntT.unbind() d = unbound[0] @@ -1686,6 +1714,17 @@ class TestNestedTensorAutograd(TestCase): data = (a, b, c) assert gradcheck(grad_test_func, inputs=data, check_batched_grad=False) + def test_indexing_backward(self): + x0 = torch.randn((2, 5)) + x1 = torch.randn((3, 4)) + nt = torch.nested.nested_tensor([x0, x1], requires_grad=True) + self.assertEqual(nt[0], x0) + self.assertEqual(nt[-1], x1) + grad_x0 = torch.randn((2, 5)) + nt[0].backward(grad_x0) + expected_grad = torch.nested.nested_tensor([grad_x0, torch.zeros((3, 4))]) + self.assertEqual(nt.grad, expected_grad) + instantiate_device_type_tests(TestNestedTensorDeviceType, globals()) if __name__ == '__main__': diff --git a/test/test_nn.py b/test/test_nn.py index 43f96d27558b..bfefe4f5a642 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -5867,7 +5867,7 @@ tensor(..., device='meta', size=(1,), requires_grad=True)""") def test_multihead_attn_nested_tensor_outside_fast_path(self): mha = torch.nn.MultiheadAttention(4, 4, batch_first=True).eval() - nt = torch.nested_tensor([torch.randn(4, 4)]) + nt = torch.nested.nested_tensor([torch.randn(4, 4)]) # One tested platform (linux-bionic-py3.7-clang) has a torch_function for one # or more of these. Take advantage of that to test the torch_function bailout. has_torch_func = torch.overrides.has_torch_function( @@ -5888,7 +5888,7 @@ tensor(..., device='meta', size=(1,), requires_grad=True)""") mha(nt, nt, nt) with torch.inference_mode(): mha(nt, nt, nt) - nt = torch.nested_tensor([torch.randn(4, 4, requires_grad=False)]) + nt = torch.nested.nested_tensor([torch.randn(4, 4, requires_grad=False)]) nt.requires_grad = False with self.assertRaisesRegex(AssertionError, msg): mha(nt, nt, nt) @@ -13933,10 +13933,10 @@ class TestNNDeviceType(NNTestCase): # result. with self.assertRaisesRegex( AssertionError, 'MultiheadAttention does not support NestedTensor outside'): - nt = torch.nested_tensor([], device=device) + nt = torch.nested.nested_tensor([], device=device) _test_module_empty_input(self, encoder_layer, nt, check_size=False, inference=True) - nt = torch.nested_tensor([torch.rand(0, 512, device=device)], device=device) + nt = torch.nested.nested_tensor([torch.rand(0, 512, device=device)], device=device) _test_module_empty_input(self, encoder_layer, nt, check_size=False, inference=True) else: _test_module_empty_input(self, encoder_layer, input, check_size=False) @@ -18743,7 +18743,7 @@ class TestNNDeviceType(NNTestCase): mask = torch.zeros(encoder_input.shape[:-1], device=device, dtype=torch.bool) mask[0][-1] = True - nt = torch.nested_tensor([encoder_input[0][:-1], encoder_input[1]], device=device) + nt = torch.nested.nested_tensor([encoder_input[0][:-1], encoder_input[1]], device=device) result = model(nt) ref_output = torch.tensor( [ diff --git a/test/test_torch.py b/test/test_torch.py index ae302c1a20d2..8de5b822d00a 100644 --- a/test/test_torch.py +++ b/test/test_torch.py @@ -5701,10 +5701,10 @@ class TestTorch(TestCase): self.assertFalse(t1.is_same_size(t3)) self.assertTrue(t1.is_same_size(t4)) - nt1 = torch.nested_tensor([torch.ones(2, 4), torch.ones(3, 4), torch.ones(5, 4)]) - nt2 = torch.nested_tensor([torch.ones(2, 4), torch.ones(2, 4), torch.ones(2, 4)]) - nt3 = torch.nested_tensor([torch.ones(2, 4, 5), torch.ones(2, 6, 5)]) - nt4 = torch.nested_tensor([torch.ones(2, 4), torch.ones(3, 4), torch.ones(5, 4)]) + nt1 = torch.nested.nested_tensor([torch.ones(2, 4), torch.ones(3, 4), torch.ones(5, 4)]) + nt2 = torch.nested.nested_tensor([torch.ones(2, 4), torch.ones(2, 4), torch.ones(2, 4)]) + nt3 = torch.nested.nested_tensor([torch.ones(2, 4, 5), torch.ones(2, 6, 5)]) + nt4 = torch.nested.nested_tensor([torch.ones(2, 4), torch.ones(3, 4), torch.ones(5, 4)]) self.assertFalse(nt1.is_same_size(nt2)) self.assertFalse(nt1.is_same_size(nt3)) diff --git a/test/test_transformers.py b/test/test_transformers.py index 16724df423b4..b3a166bf6bc4 100644 --- a/test/test_transformers.py +++ b/test/test_transformers.py @@ -844,7 +844,7 @@ class TestTransformers(NNTestCase): aligned_mask = torch.Tensor([[0, 0, 1]]).to(torch.bool) not_aligned_mask = torch.Tensor([[1, 0, 1]]).to(torch.bool) - nested_tensor_return_value = torch.nested_tensor([torch.ones((2, 2), dtype=torch.float)]) + nested_tensor_return_value = torch.nested.nested_tensor([torch.ones((2, 2), dtype=torch.float)]) tensor_return_value = torch.ones((1, 3, 2), dtype=torch.float) # Left aligned mask results in sparsity fastpath diff --git a/tools/autograd/derivatives.yaml b/tools/autograd/derivatives.yaml index 4b70209ce5ee..749f1c42a95d 100644 --- a/tools/autograd/derivatives.yaml +++ b/tools/autograd/derivatives.yaml @@ -2710,7 +2710,7 @@ self: grad.reshape(self.sizes()) # Nested Tensor -- name: nested_tensor(Tensor[] list, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor +- name: _nested_tensor_from_tensor_list(Tensor[] list, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor list: "grad.defined()? at::unbind(grad) : std::vector(list.size())" - name: _nested_tensor_from_mask(Tensor t, Tensor mask, bool mask_check=True) -> Tensor diff --git a/torch/_subclasses/fake_tensor.py b/torch/_subclasses/fake_tensor.py index 00793aaf4947..2264ae275c05 100644 --- a/torch/_subclasses/fake_tensor.py +++ b/torch/_subclasses/fake_tensor.py @@ -45,8 +45,8 @@ class DataDependentOutputException(RuntimeError): _device_not_kwarg_ops = ( aten._resize_output_.default, - aten.nested_tensor.default, - aten.nested_tensor.out, + aten._nested_tensor_from_tensor_list.default, + aten._nested_tensor_from_tensor_list.out, aten.pin_memory.default, aten.is_pinned.default, aten.to.device, diff --git a/torch/csrc/api/include/torch/nested.h b/torch/csrc/api/include/torch/nested.h index 9ea4989a0742..1730583d5e14 100644 --- a/torch/csrc/api/include/torch/nested.h +++ b/torch/csrc/api/include/torch/nested.h @@ -6,6 +6,44 @@ namespace torch { namespace nested { +/// Nested tensor +/// +/// See +/// https://pytorch.org/docs/master/nested.html#torch.nested.nested_tensor +/// +/// ``` +inline Tensor nested_tensor( + TensorList list, + c10::optional dtype = c10::nullopt, + c10::optional device = c10::nullopt, + c10::optional requires_grad = false, + c10::optional pin_memory = false) { + std::vector new_list; + for (const auto i : c10::irange(list.size())) { + new_list.push_back(list[i].clone().detach()); + } + auto out = torch::_nested_tensor_from_tensor_list( + new_list, dtype, c10::nullopt, device, pin_memory); + if (requires_grad.has_value() && requires_grad.value()) { + out.requires_grad_(true); + } + return out; +} + +/// As Nested Tensor +/// +/// See +/// https://pytorch.org/docs/master/nested.html#torch.nested.as_nested_tensor +/// +/// ``` +inline Tensor as_nested_tensor( + TensorList list, + c10::optional dtype = c10::nullopt, + c10::optional device = c10::nullopt) { + return at::_nested_tensor_from_tensor_list( + list, dtype, c10::nullopt, device, c10::nullopt); +} + /// Nested to padded tensor /// /// See diff --git a/torch/nested/__init__.py b/torch/nested/__init__.py index 3c888e0a837f..34db2d49c37a 100644 --- a/torch/nested/__init__.py +++ b/torch/nested/__init__.py @@ -1,11 +1,87 @@ +from typing import List, Optional import torch from torch._C import _add_docstr, _nested # type: ignore[attr-defined] +from torch import Tensor + +from torch.types import _dtype as DType +from torch.types import _device as Device __all__ = [ - 'to_padded_tensor' + 'to_padded_tensor', + 'as_nested_tensor', + 'nested_tensor', ] -Tensor = torch.Tensor +# Nested Tensor constructor functions +# TODO: move these to pybind to accept numpy/nested lists as inputs in the future +def nested_tensor(tensor_list: List[Tensor], *, dtype: Optional[DType] = None, device: Optional[Device] = None, + requires_grad: Optional[bool] = False, pin_memory: Optional[bool] = False) -> Tensor: + r""" + Constructs a nested tensor with no autograd history (also known as a “leaf tensor”, see + :ref:`Autograd mechanics `) from :attr:`tensor_list` a list of tensors. + + Args: + tensor_list (List[Tensor]): a list of tensors with the same ndim + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + requires_grad (bool, optional): If autograd should record operations on the + returned nested tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned nested tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.nested_tensor([a, b], requires_grad=True) + >>> nt.is_leaf + True + """ + if not isinstance(tensor_list, list) or any([not torch.is_tensor(t) for t in tensor_list]): + raise TypeError("nested_tensor(): Expected first argument to be a list of tensors ") + new_data = [t.detach() for t in tensor_list] + nt = torch._nested_tensor_from_tensor_list(new_data, dtype, None, device, pin_memory) + if (requires_grad): + nt.requires_grad_(requires_grad) + return nt + +def as_nested_tensor(tensor_list: List[Tensor], dtype: Optional[DType] = None, device: Optional[Device] = None) -> Tensor: + r""" + Constructs a nested tensor preserving autograd history from :attr:`tensor_list` a list of tensors. + + .. note:: + Tensors within the list are always copied by this function due to current nested tensor semantics. + + Args: + tensor_list (List[Tensor]): a list of tensors with the same ndim + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.as_nested_tensor([a, b]) + >>> nt.is_leaf + False + >>> fake_grad = torch.nested_tensor([torch.ones_like(a), torch.zeros_like(b)]) + >>> nt.backward(fake_grad) + >>> a.grad + tensor([1., 1., 1.]) + >>> b.grad + tensor([0., 0., 0., 0., 0.]) + """ + if not isinstance(tensor_list, list) or any([not torch.is_tensor(t) for t in tensor_list]): + raise TypeError("nested_tensor(): Expected first argument to be a list of tensors ") + return torch._nested_tensor_from_tensor_list(tensor_list, dtype, None, device, None) # Note: This not only adds doc strings for the nested ops, but # also connects the torch.nested Python namespace to the torch._C._nested builtins. @@ -14,7 +90,7 @@ to_padded_tensor = _add_docstr(_nested.nested_to_padded_tensor, r""" to_padded_tensor(input, padding, output_size=None, out=None) -> Tensor -Returns a new (non-nested) Tensor by padding the nested tensor. +Returns a new (non-nested) Tensor by padding the attr:`input` nested tensor. The leading entries will be filled with the nested data, while the trailing entries will be padded. @@ -34,7 +110,7 @@ Keyword args: Example:: - >>> nt = torch.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]) + >>> nt = torch.nested.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]) nested_tensor([ tensor([[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276], [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995]]), diff --git a/torch/overrides.py b/torch/overrides.py index 11e5c3a7ffdf..33f339c60012 100644 --- a/torch/overrides.py +++ b/torch/overrides.py @@ -160,7 +160,6 @@ def get_ignored_functions() -> Set[Callable]: torch.mkldnn_max_pool2d, torch.mkldnn_max_pool3d, torch.mkldnn_linear_backward_weights, - torch.nested_tensor, torch.normal, torch.ones, torch.promote_types,