mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
df741c589f
Syncing nvfuser devel branch to upstream master. https://github.com/csarofeen/pytorch/ Code changes includes: - codegen improvements: 1. removes un-necessary sync from redundant thread compute analysis 2. symmetric API for BestEffortReplay 3. support merge on trivial reductions 4. Ampere async copy improvements - bug fixes: 1. vectorization bug fixes 2. type inference patch : fixes upstream #81725 3. segmenter bug fix with deterministic iteration ordering - parser update 1. added leaky_relu - scheduler 1. normalization scheduler clean up. 2. simplifies matmul scheduling with new transform propagator 3. merge all dimensions in PW scheduler 4. various gemm related improvements - debuggability 1. nsight compute support 2. debug dump for InlinePropagator 3. Add `UnaryOpType::Print` Squashed commits to WAR github API Commits that's actually in this PR from the devel branch: ```dfe02f3faeMerge remote-tracking branch 'csarofeen/devel' into HEAD16173732ecAdd `TensorViewBuilder::shape(std::vector<Val*> shape)` (#1884)7cfb7796bdMerge pull request #1887 from csarofeen/upstream_merge_08033399f6de62Merge remote-tracking branch 'origin/viable/strict' into HEAD01208f5bbaAdd `UnaryOpType::Print` which can be helpful for debugging (#1878)0646522454Remove redundant TORCH_INTERNAL_ASSERT in lower_magic_zero.cpp (#1881)7bc76aa219Fix most inlined propagator for mismatched dims (#1875)501f4aa270Nonaffine swizzle formulation ep.2: Loop swizzle variant. (#1826)d863d690f9Ampere async copy ep.2: circular buffering extension to support pipelined matmul operand load (#1827)e0ae11a61cLarger sized mma instructions to support full vectorization (#1824)9bb4cf7a66fragment iteration to support fully unrolled mma ops (#1823)a48270a18dMerge all dims in pointwise scheduler (#1872)172fb3673fMake MostInlined and BestEffort inline propagation no longer assert replayed (#1868)a64462a5acAllow trivial reduction to be merged (#1871)440102bcdaSymmetric API for BestEffortReplay (#1870)d1caf330c0Some misc cleanups/refactor split out from #1854 (#1867)1013eda50bRemove some welford specific logic. (#1864)51589d36beSome cleanups on tests and heuristics params (#1866)a6b3e70da5Segmenter bug fix, and deterministic iteration ordering. (#1865)1b665b9b5eAdd nullptr checks to IrBuilder (#1861)1cd9451d74Simplify matmul scheduling with the new transform propagator. (#1817)bbc1fb9b8cAdd leaky_relu operation (#1852)e842a9bab5Minor cleanup in pointwise scheduler (#1858)9ee850ca2fFix stringstream usage (#1857)20a36c1e4fImprove nsight compute support (#1855)4059103083Remove debugging `true ||` from getPointwiseHeuristics (#1822)01117bfe8fMisc cleanup (#1853)5cc64943dcApply the magic-zero protection to each indexed domain individually for predicate indexing (#1846)92e6f0207eCleanup normalization scheduler (#1845)db89c6591aType inference patch (#1848)102fe93a46Add debug dump for InlinePropagator (#1847)b7a4d93d37Redundant thread compute analysis to avoid un-necessary sync insertion (#1687)942be5b256Upstream ci build fixes (#1842)0b83645915Fix vectorization bug introduced in #1831 (#1840)63630f1ae0Move MaxProducerPosUpdater into InlinePropagator::tearDown (#1825)9135a963c0Fix transpose benchmark dtype (#1839)2c9a6c0231Add extra configurability to `parallelizeAllLike` (#1831) ``` RUN_TORCHBENCH: nvfuser Differential Revision: [D38543000](https://our.internmc.facebook.com/intern/diff/D38543000) Pull Request resolved: https://github.com/pytorch/pytorch/pull/83067 Approved by: https://github.com/davidberard98
477 lines
17 KiB
C++
477 lines
17 KiB
C++
#include <torch/csrc/jit/codegen/cuda/arith.h>
|
|
#include <torch/csrc/jit/codegen/cuda/executor.h>
|
|
#include <torch/csrc/jit/codegen/cuda/fusion.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_all_nodes.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_utils.h>
|
|
#include <torch/csrc/jit/codegen/cuda/lower2device.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ops/all_ops.h>
|
|
#include <torch/csrc/jit/codegen/cuda/scheduler/all_schedulers.h>
|
|
|
|
#include <benchmark/benchmark.h>
|
|
|
|
#include <cuda_runtime.h>
|
|
|
|
#include <benchmarks/cpp/nvfuser/utils.h>
|
|
|
|
#define TRANSPOSE_CONFIG {true, false, false, false}
|
|
|
|
using namespace torch::jit::fuser::cuda;
|
|
|
|
struct TransposeConfig {
|
|
bool input1_transpose_axes = false;
|
|
bool input2_transpose_axes = false;
|
|
bool intermediate_transpose_axes = false;
|
|
bool output_transpose_axes = false;
|
|
};
|
|
|
|
std::vector<at::Tensor> generateInputs(
|
|
DataType dtype,
|
|
int num_dims,
|
|
std::pair<int, int> axes,
|
|
int perm_size,
|
|
int innerdim_size,
|
|
bool input1_transpose_axes,
|
|
bool input2_transpose_axes,
|
|
bool non_vectorize_offset = false,
|
|
int iter_size = 32) {
|
|
at::manual_seed(0);
|
|
auto options =
|
|
at::TensorOptions().dtype(data_type_to_aten(dtype)).device(at::kCUDA, 0);
|
|
|
|
std::vector<int64_t> transpose_shape(num_dims, iter_size);
|
|
transpose_shape[axes.second] = innerdim_size;
|
|
transpose_shape[axes.first] = perm_size;
|
|
|
|
std::vector<int64_t> non_transpose_shape(num_dims, iter_size);
|
|
non_transpose_shape[axes.first] = innerdim_size;
|
|
non_transpose_shape[axes.second] = perm_size;
|
|
|
|
// TensorType: Concrete, Contig, Symbolic
|
|
// Vectorization | Unroll - Add 1 to sizes
|
|
// Shift axis by 1 to disable vectorize loads
|
|
if (non_vectorize_offset) {
|
|
for (auto idx : c10::irange(transpose_shape.size())) {
|
|
transpose_shape[idx] += 1;
|
|
}
|
|
for (auto idx : c10::irange(non_transpose_shape.size())) {
|
|
non_transpose_shape[idx] += 1;
|
|
}
|
|
}
|
|
|
|
auto optionalTransposeSize =
|
|
[&transpose_shape, &non_transpose_shape](bool transpose_tensor) {
|
|
return (transpose_tensor) ? transpose_shape : non_transpose_shape;
|
|
};
|
|
|
|
at::Tensor aten_input1 =
|
|
at::randn(optionalTransposeSize(input1_transpose_axes), options);
|
|
at::Tensor aten_input2 =
|
|
at::randn(optionalTransposeSize(input2_transpose_axes), options);
|
|
return {aten_input1, aten_input2};
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
static void setupTranspose(
|
|
Fusion* fusion,
|
|
DataType dtype,
|
|
int num_dims,
|
|
std::pair<int, int> axes,
|
|
TransposeConfig tc) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto optionalTranspose = [axes](TensorView* tv, bool is_transpose) {
|
|
return (is_transpose) ? transpose(tv, axes.first, axes.second) : tv;
|
|
};
|
|
|
|
auto input1 = makeContigTensor(num_dims, dtype);
|
|
auto input2 = makeContigTensor(num_dims, dtype);
|
|
fusion->addInput(input1);
|
|
fusion->addInput(input2);
|
|
|
|
auto ot_input1 = optionalTranspose(input1, tc.input1_transpose_axes);
|
|
auto ot_input2 = optionalTranspose(input2, tc.input2_transpose_axes);
|
|
auto intermediate = add(ot_input1, ot_input2);
|
|
auto ot_intermediate =
|
|
optionalTranspose(intermediate, tc.intermediate_transpose_axes);
|
|
auto output = relu(ot_intermediate);
|
|
auto ot_output = optionalTranspose(output, tc.output_transpose_axes);
|
|
fusion->addOutput(ot_output);
|
|
}
|
|
|
|
static void NvFuserScheduler_Transpose(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
DataType dtype,
|
|
int num_dims,
|
|
std::pair<int, int> axes,
|
|
TransposeConfig tc) {
|
|
auto aten_inputs = generateInputs(
|
|
dtype,
|
|
num_dims,
|
|
axes,
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
tc.input1_transpose_axes,
|
|
tc.input2_transpose_axes);
|
|
auto at_input1 = aten_inputs[0];
|
|
auto at_input2 = aten_inputs[1];
|
|
|
|
std::vector<c10::IValue> fuser_inputs = {at_input1, at_input2};
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, fuser_inputs);
|
|
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
((at_input1.numel() * 3) * int64_t(dataTypeSize(dtype))));
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
#define NVFUSER_TRANSPOSE_SQUARE_RUN( \
|
|
TITLE, DTYPE, NUM_DIMS, AXIS1, AXIS2, CONFIG) \
|
|
NVFUSER_BENCHMARK_DEFINE( \
|
|
TITLE, \
|
|
setupTranspose, \
|
|
NvFuserScheduler_Transpose, \
|
|
DTYPE, \
|
|
NUM_DIMS, \
|
|
{AXIS1, AXIS2}, \
|
|
CONFIG); \
|
|
\
|
|
NVFUSER_BENCHMARK_RUN(TITLE) \
|
|
->RangeMultiplier(8) \
|
|
->Args({9, 2408}) \
|
|
->Args({16, 512}) \
|
|
->Args({18, 96}) \
|
|
->Args({24, 96}) \
|
|
->Args({24, 256}) \
|
|
->Args({24, 512}) \
|
|
->Args({32, 27}) \
|
|
->Args({32, 96}) \
|
|
->Args({32, 288}) \
|
|
->Args({32, 864}) \
|
|
->Args({40, 120}) \
|
|
->Args({48, 128}) \
|
|
->Args({48, 256}) \
|
|
->Args({49, 512}) \
|
|
->Args({49, 1024}) \
|
|
->Args({49, 2048}) \
|
|
->Args({49, 4608}) \
|
|
->Args({64, 64}) \
|
|
->Args({64, 96}) \
|
|
->Args({64, 128}) \
|
|
->Args({64, 147}) \
|
|
->Args({64, 192}) \
|
|
->Args({64, 256}) \
|
|
->Args({64, 288}) \
|
|
->Args({64, 512}) \
|
|
->Args({80, 64}) \
|
|
->Args({81, 1728}) \
|
|
->Args({83, 1728}) \
|
|
->Args({96, 864}) \
|
|
->Args({100, 1280}) \
|
|
->Args({100, 4032}) \
|
|
->Args({120, 40}) \
|
|
->Args({128, 128}) \
|
|
->Args({128, 512}) \
|
|
->Args({128, 1152}) \
|
|
->Args({192, 128}) \
|
|
->Args({192, 256}) \
|
|
->Args({192, 720}) \
|
|
->Args({192, 768}) \
|
|
->Args({192, 1120}) \
|
|
->Args({192, 1728}) \
|
|
->Args({196, 256}) \
|
|
->Args({196, 512}) \
|
|
->Args({196, 1024}) \
|
|
->Args({196, 2304}) \
|
|
->Args({256, 256}) \
|
|
->Args({256, 1024}) \
|
|
->Args({256, 2304}) \
|
|
->Args({284, 512}) \
|
|
->Args({320, 1280}) \
|
|
->Args({320, 1728}) \
|
|
->Args({324, 2592}) \
|
|
->Args({361, 768}) \
|
|
->Args({361, 1120}) \
|
|
->Args({384, 2}) \
|
|
->Args({384, 32}) \
|
|
->Args({384, 128}) \
|
|
->Args({384, 256}) \
|
|
->Args({384, 512}) \
|
|
->Args({384, 1280}) \
|
|
->Args({384, 2592}) \
|
|
->Args({384, 4032}) \
|
|
->Args({448, 1280}) \
|
|
->Args({480, 16}) \
|
|
->Args({480, 256}) \
|
|
->Args({512, 2}) \
|
|
->Args({512, 16}) \
|
|
->Args({512, 128}) \
|
|
->Args({512, 256}) \
|
|
->Args({512, 1024}) \
|
|
->Args({512, 2048}) \
|
|
->Args({512, 3072}) \
|
|
->Args({512, 4608}) \
|
|
->Args({784, 40}) \
|
|
->Args({784, 120}) \
|
|
->Args({784, 128}) \
|
|
->Args({784, 1152}) \
|
|
->Args({1001, 2408}) \
|
|
->Args({1024, 16}) \
|
|
->Args({1024, 256}) \
|
|
->Args({1024, 512}) \
|
|
->Args({1024, 1024}) \
|
|
->Args({1024, 3072}) \
|
|
->Args({1369, 192}) \
|
|
->Args({1369, 256}) \
|
|
->Args({1369, 288}) \
|
|
->Args({2048, 512}) \
|
|
->Args({2048, 1024}) \
|
|
->Args({2250, 27}) \
|
|
->Args({3072, 512}) \
|
|
->Args({3072, 1024}) \
|
|
->Args({3136, 64}) \
|
|
->Args({5329, 720}) \
|
|
->Args({5625, 64}) \
|
|
->Args({12544, 147}) \
|
|
->Args({22201, 288}) \
|
|
->Unit(benchmark::kMicrosecond)
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp32_Inner_2D_01_Axis,
|
|
DataType::Float,
|
|
2 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp32_Inner_3D_02_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp32_Inner_3D_12_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
1 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp32_Outer_3D_01_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp16_Inner_2D_01_Axis,
|
|
DataType::Half,
|
|
2 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp16_Inner_3D_02_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp16_Inner_3D_12_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
1 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_SQUARE_RUN(
|
|
NF_Transpose_Random_fp16_Outer_3D_01_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
|
|
#define NVFUSER_TRANSPOSE_RUN(TITLE, DTYPE, NUM_DIMS, AXIS1, AXIS2, CONFIG) \
|
|
NVFUSER_BENCHMARK_DEFINE( \
|
|
TITLE, \
|
|
setupTranspose, \
|
|
NvFuserScheduler_Transpose, \
|
|
DTYPE, \
|
|
NUM_DIMS, \
|
|
{AXIS1, AXIS2}, \
|
|
CONFIG); \
|
|
\
|
|
NVFUSER_BENCHMARK_RUN(TITLE) \
|
|
->RangeMultiplier(8) \
|
|
->Ranges({{2, 256 * 256}, {160, 320}}) \
|
|
->Unit(benchmark::kMicrosecond) \
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp32_Inner_2D_01_Axis,
|
|
DataType::Float,
|
|
2 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp32_Inner_3D_02_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp32_Inner_3D_12_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
1 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp32_Outer_3D_01_Axis,
|
|
DataType::Float,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp16_Inner_2D_01_Axis,
|
|
DataType::Half,
|
|
2 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp16_Inner_3D_02_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp16_Inner_3D_12_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
1 /* axis1 */,
|
|
2 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
NVFUSER_TRANSPOSE_RUN(
|
|
NF_Transpose_fp16_Outer_3D_01_Axis,
|
|
DataType::Half,
|
|
3 /* num_dims */,
|
|
0 /* axis1 */,
|
|
1 /* axis2 */,
|
|
TransposeConfig(TRANSPOSE_CONFIG));
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
static void Baseline_Transpose(
|
|
benchmark::State& benchmark_state,
|
|
DataType dtype,
|
|
int num_dims,
|
|
std::pair<int, int> axes,
|
|
TransposeConfig tc) {
|
|
auto aten_inputs = generateInputs(
|
|
dtype,
|
|
num_dims,
|
|
axes,
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
tc.input1_transpose_axes,
|
|
tc.input2_transpose_axes);
|
|
auto at_input1 = aten_inputs[0];
|
|
auto at_input2 = aten_inputs[1];
|
|
|
|
auto optionalTransposeAten = [&axes](at::Tensor x, bool is_transpose) {
|
|
return (is_transpose) ? at::transpose(x, axes.first, axes.second) : x;
|
|
};
|
|
|
|
for (auto _ : benchmark_state) {
|
|
clearL2Cache();
|
|
CudaKernelTimer timer;
|
|
|
|
auto at_ot_input1 =
|
|
optionalTransposeAten(at_input1, tc.input1_transpose_axes);
|
|
auto at_ot_input2 =
|
|
optionalTransposeAten(at_input2, tc.input2_transpose_axes);
|
|
auto at_intermediate = add(at_ot_input1, at_ot_input2);
|
|
auto at_ot_intermediate =
|
|
optionalTransposeAten(at_intermediate, tc.intermediate_transpose_axes);
|
|
auto at_output = relu(at_ot_intermediate);
|
|
auto at_ot_output =
|
|
optionalTransposeAten(at_output, tc.output_transpose_axes);
|
|
|
|
benchmark_state.SetIterationTime(timer.elapsed() / 1000.0);
|
|
}
|
|
// Sync everything up before we're finished, don't want to run ahead on the
|
|
// cpu while benchmarking.
|
|
cudaDeviceSynchronize();
|
|
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
(at_input1.numel() * 3 * int64_t(dataTypeSize(dtype))));
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
static void Baseline_Transpose_fp32_Inner_2D_01_Axis(
|
|
benchmark::State& benchmark_state) {
|
|
Baseline_Transpose(
|
|
benchmark_state,
|
|
DataType::Float,
|
|
2 /* num_dims */,
|
|
{0, 1} /* axes */,
|
|
TRANSPOSE_CONFIG);
|
|
}
|
|
|
|
static void Baseline_Transpose_fp16_Inner_2D_01_Axis(
|
|
benchmark::State& benchmark_state) {
|
|
Baseline_Transpose(
|
|
benchmark_state,
|
|
DataType::Half,
|
|
2 /* num_dims */,
|
|
{0, 1} /* axes */,
|
|
TRANSPOSE_CONFIG);
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
BENCHMARK(Baseline_Transpose_fp32_Inner_2D_01_Axis)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
BENCHMARK(Baseline_Transpose_fp16_Inner_2D_01_Axis)
|
|
// ->RangeMultiplier(2)
|
|
->Ranges({{2, 1024 * 1024}, {160, 320}})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
//------------------------------------------------------------------------------
|