From a7f4f80903b331ecb9485ccf534df1dfc5748cff Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Mon, 24 May 2021 13:46:55 -0700 Subject: [PATCH] ENH Adds dtype to nn.functional.one_hot (#58090) Summary: Fixes https://github.com/pytorch/pytorch/issues/33046 Related to https://github.com/pytorch/pytorch/issues/53785 Pull Request resolved: https://github.com/pytorch/pytorch/pull/58090 Reviewed By: zou3519 Differential Revision: D28640893 Pulled By: jbschlosser fbshipit-source-id: 3686579517ccc75beaa74f0f6d167f5e40a83fd2 --- aten/src/ATen/native/Onehot.cpp | 7 ++-- aten/src/ATen/native/native_functions.yaml | 2 +- test/cpp/api/functional.cpp | 11 ++++++ test/onnx/test_pytorch_onnx_onnxruntime.py | 23 +++++++----- test/test_nn.py | 35 ++++++++++--------- tools/autograd/derivatives.yaml | 2 +- .../include/torch/nn/functional/embedding.h | 7 ++-- torch/csrc/utils/python_arg_parser.cpp | 2 +- torch/nn/functional.py | 4 ++- torch/onnx/symbolic_opset9.py | 10 ++++-- 10 files changed, 65 insertions(+), 38 deletions(-) diff --git a/aten/src/ATen/native/Onehot.cpp b/aten/src/ATen/native/Onehot.cpp index 6893525fedc8..1bf75ac46f66 100644 --- a/aten/src/ATen/native/Onehot.cpp +++ b/aten/src/ATen/native/Onehot.cpp @@ -2,9 +2,10 @@ namespace at { namespace native { -Tensor one_hot(const Tensor &self, int64_t num_classes) { +Tensor one_hot(const Tensor &self, int64_t num_classes, ScalarType dtype) { TORCH_CHECK(self.dtype() == kLong, "one_hot is only applicable to index tensor."); auto shape = self.sizes().vec(); + auto output_options = self.options().dtype(dtype); // empty tensor could be converted to one hot representation, // but shape inference is not possible. @@ -13,7 +14,7 @@ Tensor one_hot(const Tensor &self, int64_t num_classes) { AT_ERROR("Can not infer total number of classes from empty tensor."); } else { shape.push_back(num_classes); - return at::empty(shape, self.options()); + return at::empty(shape, output_options); } } @@ -35,7 +36,7 @@ Tensor one_hot(const Tensor &self, int64_t num_classes) { } shape.push_back(num_classes); - Tensor ret = at::zeros(shape, self.options()); + Tensor ret = at::zeros(shape, output_options); ret.scatter_(-1, self.unsqueeze(-1), 1); return ret; } diff --git a/aten/src/ATen/native/native_functions.yaml b/aten/src/ATen/native/native_functions.yaml index ef197ab46e5e..ff1f5a15393a 100644 --- a/aten/src/ATen/native/native_functions.yaml +++ b/aten/src/ATen/native/native_functions.yaml @@ -4112,7 +4112,7 @@ dispatch: MkldnnCPU: mkldnn_transpose_ -- func: one_hot(Tensor self, int num_classes=-1) -> Tensor +- func: one_hot(Tensor self, int num_classes=-1, ScalarType dtype=long) -> Tensor python_module: nn variants: function diff --git a/test/cpp/api/functional.cpp b/test/cpp/api/functional.cpp index 768f3c45fb89..f920e49cfb18 100644 --- a/test/cpp/api/functional.cpp +++ b/test/cpp/api/functional.cpp @@ -1061,6 +1061,17 @@ TEST_F(FunctionalTest, OneHot) { ASSERT_TRUE(torch::allclose(y, expected)); ASSERT_EQ(y.sizes(), std::vector({3, 2, 3})); } + + { // Test #4 + auto x = torch::arange(0, 5, torch::kLong); + auto y = F::one_hot(x % 3, 3, torch::kUInt8); + auto expected = torch::tensor( + {{1, 0, 0}, {0, 1, 0}, {0, 0, 1}, {1, 0, 0}, {0, 1, 0}}, torch::kUInt8); + + ASSERT_EQ(y.ndimension(), 2); + ASSERT_TRUE(torch::allclose(y, expected)); + ASSERT_EQ(y.sizes(), std::vector({5, 3})); + } } // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/test/onnx/test_pytorch_onnx_onnxruntime.py b/test/onnx/test_pytorch_onnx_onnxruntime.py index c52b1303c82b..e1df640c4055 100644 --- a/test/onnx/test_pytorch_onnx_onnxruntime.py +++ b/test/onnx/test_pytorch_onnx_onnxruntime.py @@ -81,12 +81,15 @@ def run_ort(ort_sess, input): return inline_flatten_list(ort_outs, []) -def ort_compare_with_pytorch(ort_outs, output, rtol, atol): +def ort_compare_with_pytorch(ort_outs, output, rtol, atol, check_dtypes=False): output, _ = torch.jit._flatten(output) outputs = [to_numpy(outp) for outp in output] # compare onnxruntime and PyTorch results assert len(outputs) == len(ort_outs), "number of outputs differ" + if check_dtypes: + assert all(l.dtype == r.dtype for l, r in zip(outputs, ort_outs)), \ + "dtypes of outputs differ" # compare onnxruntime and PyTorch results [np.testing.assert_allclose(out, ort_out, rtol=rtol, atol=atol) for out, ort_out in zip(outputs, ort_outs)] @@ -98,7 +101,7 @@ def run_model_test(self, model, batch_size=2, state_dict=None, dynamic_axes=None, test_with_inputs=None, input_names=None, output_names=None, fixed_batch_size=False, dict_check=True, - training=None): + training=None, check_dtypes=False): model.eval() if input is None: input = torch.randn(batch_size, 3, 224, 224, requires_grad=True) @@ -133,7 +136,7 @@ def run_model_test(self, model, batch_size=2, state_dict=None, onnx_shape_inference=self.onnx_shape_inference) # compute onnxruntime output prediction ort_outs = run_ort(ort_sess, input) - ort_compare_with_pytorch(ort_outs, output, rtol, atol) + ort_compare_with_pytorch(ort_outs, output, rtol, atol, check_dtypes) # if additional test inputs are provided run the onnx @@ -147,7 +150,7 @@ def run_model_test(self, model, batch_size=2, state_dict=None, if isinstance(output, torch.Tensor): output = (output,) ort_outs = run_ort(ort_sess, test_input) - ort_compare_with_pytorch(ort_outs, output, rtol, atol) + ort_compare_with_pytorch(ort_outs, output, rtol, atol, check_dtypes) def _init_test_generalized_rcnn_transform(): min_size = 100 @@ -240,7 +243,7 @@ class TestONNXRuntime(unittest.TestCase): def run_test(self, model, input, rtol=1e-3, atol=1e-7, do_constant_folding=True, batch_size=2, use_gpu=True, dynamic_axes=None, test_with_inputs=None, input_names=None, output_names=None, fixed_batch_size=False, dict_check=True, - training=None): + training=None, check_dtypes=False): def _run_test(m): return run_model_test(self, m, batch_size=batch_size, input=input, use_gpu=use_gpu, rtol=rtol, atol=atol, @@ -248,7 +251,7 @@ class TestONNXRuntime(unittest.TestCase): dynamic_axes=dynamic_axes, test_with_inputs=test_with_inputs, input_names=input_names, output_names=output_names, fixed_batch_size=fixed_batch_size, dict_check=dict_check, - training=training) + training=training, check_dtypes=check_dtypes) if self.is_script_test_enabled: script_model = torch.jit.script(model) _run_test(script_model) @@ -3256,15 +3259,17 @@ class TestONNXRuntime(unittest.TestCase): @skipIfUnsupportedMinOpsetVersion(9) def test_one_hot(self): class OneHot(torch.nn.Module): - def __init__(self, num_classes): + def __init__(self, num_classes, dtype): super().__init__() self.num_classes = num_classes + self.dtype = dtype def forward(self, x): - return torch.nn.functional.one_hot(x, self.num_classes) + return torch.nn.functional.one_hot(x, self.num_classes, self.dtype) x = torch.arange(10) - self.run_test(OneHot(15), (x)) + self.run_test(OneHot(15, torch.long), (x), check_dtypes=True) + self.run_test(OneHot(15, torch.uint8), (x), check_dtypes=True) @skipIfUnsupportedMinOpsetVersion(9) def test_gather(self): diff --git a/test/test_nn.py b/test/test_nn.py index 1a0f7700483b..913f0fb1889d 100644 --- a/test/test_nn.py +++ b/test/test_nn.py @@ -12867,55 +12867,56 @@ class TestNNDeviceType(NNTestCase): inp = torch.randn(0, 7, device=device) self._test_module_empty_input(mod, inp) - def test_one_hot(self, device): + @dtypes(torch.uint8, torch.long, torch.bool) + def test_one_hot(self, device, dtype): if self.device_type != 'cuda': # cuda throws device assert for invalid data with self.assertRaises(RuntimeError): - torch.nn.functional.one_hot(torch.tensor([3, 4, -1, 0], device=device), -1) + torch.nn.functional.one_hot(torch.tensor([3, 4, -1, 0], device=device, dtype=dtype), -1) with self.assertRaises(RuntimeError): - torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), 3) + torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device, dtype=dtype), 3) - t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device)) + t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), dtype=dtype) expected = torch.tensor([[0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], - [1, 0, 0, 0, 0]], device=device) + [1, 0, 0, 0, 0]], device=device, dtype=dtype) self.assertEqual(t, expected) - t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), -1) + t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), -1, dtype=dtype) expected = torch.tensor([[0, 0, 0, 1, 0], [0, 0, 0, 0, 1], [0, 1, 0, 0, 0], - [1, 0, 0, 0, 0]], device=device) + [1, 0, 0, 0, 0]], device=device, dtype=dtype) self.assertEqual(t, expected) - t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), 6) + t = torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), 6, dtype=dtype) expected = torch.tensor([[0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 1, 0], [0, 1, 0, 0, 0, 0], - [1, 0, 0, 0, 0, 0]], device=device) + [1, 0, 0, 0, 0, 0]], device=device, dtype=dtype) self.assertEqual(t, expected) - t = torch.nn.functional.one_hot(torch.tensor([[3, 4], [1, 0]], device=device)) + t = torch.nn.functional.one_hot(torch.tensor([[3, 4], [1, 0]], device=device), dtype=dtype) expected = torch.tensor([[[0, 0, 0, 1, 0], [0, 0, 0, 0, 1]], [[0, 1, 0, 0, 0], - [1, 0, 0, 0, 0]]], device=device) + [1, 0, 0, 0, 0]]], device=device, dtype=dtype) self.assertEqual(t, expected) - t = torch.nn.functional.one_hot(torch.tensor(4, device=device)) - expected = torch.tensor([0, 0, 0, 0, 1], device=device) + t = torch.nn.functional.one_hot(torch.tensor(4, device=device), dtype=dtype) + expected = torch.tensor([0, 0, 0, 0, 1], device=device, dtype=dtype) self.assertEqual(t, expected) - t = torch.nn.functional.one_hot(torch.empty([4, 0], dtype=torch.long, device=device), 100) - expected = torch.empty([4, 0, 100], dtype=torch.long) + t = torch.nn.functional.one_hot(torch.empty([4, 0], dtype=torch.long, device=device), 100, dtype=dtype) + expected = torch.empty([4, 0, 100], dtype=dtype) self.assertEqual(t, expected) with self.assertRaises(RuntimeError): - torch.nn.functional.one_hot(torch.empty([4, 0], dtype=torch.long, device=device)) + torch.nn.functional.one_hot(torch.empty([4, 0], dtype=torch.long, device=device), dtype=dtype) with self.assertRaises(RuntimeError): - torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), -2) + torch.nn.functional.one_hot(torch.tensor([3, 4, 1, 0], device=device), -2, dtype=dtype) def test_nn_scalars(self, device): # One off tests to ensure scalars from nn.yaml are properly applied diff --git a/tools/autograd/derivatives.yaml b/tools/autograd/derivatives.yaml index 16154d831a2b..54315e29fd9f 100644 --- a/tools/autograd/derivatives.yaml +++ b/tools/autograd/derivatives.yaml @@ -1154,7 +1154,7 @@ - name: t(Tensor(a) self) -> Tensor(a) self: grad.t() -- name: one_hot(Tensor self, int num_classes=-1) -> Tensor +- name: one_hot(Tensor self, int num_classes=-1, ScalarType dtype=long) -> Tensor self: non_differentiable - name: flip(Tensor self, int[] dims) -> Tensor diff --git a/torch/csrc/api/include/torch/nn/functional/embedding.h b/torch/csrc/api/include/torch/nn/functional/embedding.h index 0c8a814cd38d..f391792f2a61 100644 --- a/torch/csrc/api/include/torch/nn/functional/embedding.h +++ b/torch/csrc/api/include/torch/nn/functional/embedding.h @@ -6,8 +6,11 @@ namespace torch { namespace nn { namespace functional { -inline Tensor one_hot(const Tensor& tensor, int64_t num_classes = -1) { - return torch::one_hot(tensor, num_classes); +inline Tensor one_hot( + const Tensor& tensor, + int64_t num_classes = -1, + ScalarType dtype = c10::kLong) { + return torch::one_hot(tensor, num_classes, dtype); } #ifndef DOXYGEN_SHOULD_SKIP_THIS diff --git a/torch/csrc/utils/python_arg_parser.cpp b/torch/csrc/utils/python_arg_parser.cpp index ae9f05ab05d1..6d3031b7588a 100644 --- a/torch/csrc/utils/python_arg_parser.cpp +++ b/torch/csrc/utils/python_arg_parser.cpp @@ -675,7 +675,7 @@ void FunctionParameter::set_default_str(const std::string& str) { } else if (type_ == ParameterType::SCALARTYPE) { if (str == "None") { default_scalartype = at::ScalarType::Undefined; - } else if (str == "torch.int64") { + } else if (str == "torch.int64" || str == "at::kLong") { default_scalartype = at::ScalarType::Long; } else { throw std::runtime_error("invalid default value for ScalarType: " + str); diff --git a/torch/nn/functional.py b/torch/nn/functional.py index bcbd17f6d163..eb5fd0185559 100644 --- a/torch/nn/functional.py +++ b/torch/nn/functional.py @@ -4248,7 +4248,7 @@ Example:: one_hot = _add_docstr( torch._C._nn.one_hot, r""" -one_hot(tensor, num_classes=-1) -> LongTensor +one_hot(tensor, num_classes=-1, dtype=torch.long) -> LongTensor Takes LongTensor with index values of shape ``(*)`` and returns a tensor of shape ``(*, num_classes)`` that have zeros everywhere except where the @@ -4265,6 +4265,8 @@ Arguments: num_classes (int): Total number of classes. If set to -1, the number of classes will be inferred as one greater than the largest class value in the input tensor. + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Default: ``torch.int64``. Returns: LongTensor that has one more dimension with 1 values at the diff --git a/torch/onnx/symbolic_opset9.py b/torch/onnx/symbolic_opset9.py index 33420947861d..27ccccccf029 100644 --- a/torch/onnx/symbolic_opset9.py +++ b/torch/onnx/symbolic_opset9.py @@ -2554,11 +2554,15 @@ def prim_tolist(g, input, dim_val, elem_ty_val): return input -@parse_args('v', 'i') -def one_hot(g, self, num_classes): +@parse_args('v', 'i', 'v') +def one_hot(g, self, num_classes, dtype): values = g.op("Constant", value_t=torch.LongTensor([0, 1])) depth = g.op("Constant", value_t=torch.LongTensor([num_classes])) - return g.op("OneHot", self, depth, values, axis_i=-1) + one_hot_tensor = g.op("OneHot", self, depth, values, axis_i=-1) + dtype = sym_help._maybe_get_const(dtype, 'i') + if sym_help._is_value(dtype): + dtype = 4 # default to int64 + return g.op("Cast", one_hot_tensor, to_i=sym_help.scalar_type_to_onnx[dtype]) @parse_args('v', 'i', 'v', 'v')