mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
0e582fbfcc
Syncing nvfuser devel branch to upstream master. https://github.com/csarofeen/pytorch/ Codegen changes include: - codegen improvement: i. improved view support on pointwise and transpose scheduler ii. grouped grid welford added for better outer-norm grid persistence in normalization - misc: i. new composite ops added: variance_mean , arange, ii. fixes misaligned address for transpose scheduler iii. refactor on separation of compilation API from execution API to prepare us for async compilation iv. double type support on expression evaluator v. PYTORCH_NVFUSER_DUMP refactor to save PTX and CUBIN Commits that's in this PR from the devel branch: ```89330aa23aTensor factories must set the output shape as its input (#1939)b2fd01ea93arange support (#1933)56c00fd392Double support on all expression evaluators (#1937)371f28223eImprove trivial reduction merge support (#1931)1d0c26790eTest `rand` in a fusion with zero tensor input (#1932)0dab160fb2Fix softmax bwd sizes. (#1890)ef98f360f6Fix a bug (#1936)63132a0c56Propagate permissive mapping information into indexing pass (#1929)b4ac2c88d7Map IterationDomains through view operations. (#1919)c0a187a761do not use deprecated functions (#1935)88de85e758Upstream cherry pick fixes 0811 (#1934)b247dcf7c5Separate kernel compilation API from kernel execution API (#1914)b34e3b93eeFix `ir_utils::hasBlockSync` + misc fixes in transpose scheduler (#1924)14a53e6707Nullary RNGOp (#1892)3c3c89e638Misc fixes/tuning for transpose scheduler (#1912)20cf109c8bGrouped grid welford (#1921)6cf7eb024cTranspose scheduler small dim sizes better support (#1910)9341ea9a5bDisabled ViewPersistentShmoo sizes that results in NAN (#1922)057237f66dFix CUDA driver error: misaligned address for transpose scheduler (#1918)3fb3d80339Add variance_mean function using Welford (#1907)98febf6aa3Remove DisableOption::UnrollWithRng (#1913)ee8ef33a55Minor fix for the debug interface of using PTX directly (#1917)6e8f953351Add PYTORCH_NVFUSER_DUMP options to save PTX and CUBIN (#1916)5eefa9a723dopt is only available since nvrtc 11.7 (#1915)2ec8fc711eKill computeAtBetween (#1911)d0d106a1d9Improve view support on pointwise and transpose scheduler (#1906)e71e1ecefeFix name clash of RNG with shared memory (#1904)3381793a25Fix mutator and sameAs for expanded IterDomain (#1902) ``` RUN_TORCHBENCH: nvfuser Differential Revision: [D39324552](https://our.internmc.facebook.com/intern/diff/D39324552) Pull Request resolved: https://github.com/pytorch/pytorch/pull/84626 Approved by: https://github.com/malfet
365 lines
12 KiB
C++
365 lines
12 KiB
C++
#include <torch/csrc/jit/codegen/cuda/arith.h>
|
|
#include <torch/csrc/jit/codegen/cuda/executor.h>
|
|
#include <torch/csrc/jit/codegen/cuda/fusion.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_all_nodes.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_utils.h>
|
|
#include <torch/csrc/jit/codegen/cuda/lower2device.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ops/all_ops.h>
|
|
#include <torch/csrc/jit/codegen/cuda/scheduler/all_schedulers.h>
|
|
|
|
#include <benchmark/benchmark.h>
|
|
|
|
#include <cuda_runtime.h>
|
|
|
|
#include <benchmarks/cpp/nvfuser/utils.h>
|
|
|
|
using namespace torch::jit::fuser::cuda;
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
static void setupSoftmaxBWD(
|
|
Fusion* fusion,
|
|
DataType dtype,
|
|
const int reduction_axis) {
|
|
TORCH_INTERNAL_ASSERT(dtype == DataType::Float || dtype == DataType::Half);
|
|
|
|
FusionGuard fg(fusion);
|
|
// setup fusion
|
|
auto grad_output = makeContigTensor(2, dtype);
|
|
auto output = makeContigTensor(2, dtype);
|
|
auto input = makeContigTensor(2, dtype);
|
|
fusion->addInput(grad_output);
|
|
fusion->addInput(output);
|
|
fusion->addInput(input);
|
|
|
|
if (dtype == DataType::Half) {
|
|
grad_output = castOp(DataType::Float, grad_output);
|
|
output = castOp(DataType::Float, output);
|
|
input = castOp(DataType::Float, input);
|
|
}
|
|
|
|
auto grad_input = softmax_backward(grad_output, output, reduction_axis);
|
|
|
|
if (dtype == DataType::Half) {
|
|
grad_input = castOp(DataType::Half, grad_input);
|
|
}
|
|
|
|
fusion->addOutput(grad_input);
|
|
}
|
|
|
|
static void NvFuserScheduler_Softmax_BWD(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
DataType dtype,
|
|
const int reduction_axis) {
|
|
TORCH_INTERNAL_ASSERT(dtype == DataType::Float || dtype == DataType::Half);
|
|
|
|
at::manual_seed(0);
|
|
auto options =
|
|
at::TensorOptions().dtype(data_type_to_aten(dtype)).device(at::kCUDA, 0);
|
|
|
|
auto reduction_size = benchmark_state.range(0);
|
|
auto iter_size = benchmark_state.range(1);
|
|
|
|
at::Tensor input =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
at::Tensor grad_output =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
at::Tensor output =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
std::vector<c10::IValue> aten_inputs({grad_output, output, input});
|
|
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
(3 * input.numel() * int64_t(dataTypeSize(dtype))));
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
static void Baseline_Softmax_BWD(
|
|
benchmark::State& benchmark_state,
|
|
DataType dtype,
|
|
const int reduction_axis) {
|
|
at::manual_seed(0);
|
|
auto options =
|
|
at::TensorOptions().dtype(data_type_to_aten(dtype)).device(at::kCUDA, 0);
|
|
|
|
auto reduction_size = benchmark_state.range(0);
|
|
auto iter_size = benchmark_state.range(1);
|
|
|
|
at::Tensor input =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
at::Tensor grad_output =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
at::Tensor output =
|
|
(reduction_axis ? at::randn({iter_size, reduction_size}, options)
|
|
: at::randn({reduction_size, iter_size}, options));
|
|
|
|
for (auto _ : benchmark_state) {
|
|
clearL2Cache();
|
|
CudaKernelTimer timer;
|
|
auto grad_input = at::_softmax_backward_data(
|
|
grad_output, output, reduction_axis, data_type_to_aten(dtype));
|
|
benchmark_state.SetIterationTime(timer.elapsed() / 1000.0);
|
|
}
|
|
// Sync everything up before we're finished, don't want to run ahead on the
|
|
// cpu while benchmarking.
|
|
C10_CUDA_CHECK(cudaDeviceSynchronize());
|
|
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
(3 * input.numel() * int64_t(dataTypeSize(dtype))));
|
|
}
|
|
|
|
static void Baseline_Softmax_BWD_Outer_fp32(benchmark::State& benchmark_state) {
|
|
Baseline_Softmax_BWD(benchmark_state, DataType::Float, 0);
|
|
}
|
|
|
|
static void Baseline_Softmax_BWD_Inner_fp32(benchmark::State& benchmark_state) {
|
|
Baseline_Softmax_BWD(benchmark_state, DataType::Float, 1);
|
|
}
|
|
|
|
static void Baseline_Softmax_BWD_Outer_fp16(benchmark::State& benchmark_state) {
|
|
Baseline_Softmax_BWD(benchmark_state, DataType::Half, 0);
|
|
}
|
|
|
|
static void Baseline_Softmax_BWD_Inner_fp16(benchmark::State& benchmark_state) {
|
|
Baseline_Softmax_BWD(benchmark_state, DataType::Half, 1);
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_Softmax_BWD_Outer_fp32,
|
|
setupSoftmaxBWD,
|
|
NvFuserScheduler_Softmax_BWD,
|
|
DataType::Float,
|
|
0);
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_Softmax_BWD_Inner_fp32,
|
|
setupSoftmaxBWD,
|
|
NvFuserScheduler_Softmax_BWD,
|
|
DataType::Float,
|
|
1);
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_Softmax_BWD_Outer_fp16,
|
|
setupSoftmaxBWD,
|
|
NvFuserScheduler_Softmax_BWD,
|
|
DataType::Half,
|
|
0);
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_Softmax_BWD_Inner_fp16,
|
|
setupSoftmaxBWD,
|
|
NvFuserScheduler_Softmax_BWD,
|
|
DataType::Half,
|
|
1);
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Outer_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp32)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{1, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{32768, 16 * 1024 * 1024}, {2, 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 16}, {32768, 16 * 1024 * 1024}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Softmax_BWD_Inner_fp16)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{128, 1024 * 16}, {128, 1024 * 16}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|