mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
7b419e8513
Syncing nvfuser devel branch to upstream master. https://github.com/csarofeen/pytorch/ Codegen changes include: * codegen improvement: i. allow non-root trivial reductions, allow empty/no-op fusion ii. fixes vectorization checks and size calculation iii. bank conflict handle improvement iv. enables transpose scheduler * misc: i. CI tests failure fixes ii. cpp tests file clean up iii. trivial forwarding supports added in codegen runtime iv. added factory methods support in codegen Commits that's in this PR from the devel branch: ```7117a7e37epatching nvfuser conv cudnn test numerics mismatch (#2048)65af1a4e70Inserting sync for redundant parallel types is already done at the (#2023)6ac74d1816Fix sync map (#2047)f5bca33335Bank conflict checker improvements (#2032)d2ca7e3fd2Minor update on cp.async code generation. (#1901)d36cf61f55Test file cleanup (#2040)0b8e83f49cAllow non-root trivial reductions (#2037)a2dfe40b27Fix vectorize size calculation (#2035)e040676a31Use withPredicate to replace setPredicate to maintain Exprs immutable (#2025)197221b847removing ci workflow (#2034)40e2703d00Reduction rand like patch (#2031)bc772661cbAdd utility for checking bank conflict of shared memory (#2029)ddd1cf7695Add back FusionReductionWithTrivialReduction_CUDA (#2030)fbd97e5ef1Revert "Cleanup trivial reduction workarounds (#2006)" (#2024)bca20c1dfbCleanup trivial reduction workarounds (#2006)e4b65850eeTrivial forwarding (#1995)1a0e355b50Fix contiguity analysis of predicates to match updated contiguity. (#1991)a4effa6a5fEnable output allocation cache (#2010)35440b7953Patching bn inference (#2016)0f9f0b4060Add matmul benchmark (#2007)45045cd05eEnable tests previously disabled due to an aliasing bug (#2005)967aa77d2cContiguous indexing for View operations (#1990)a43cb20f48Make inlining even more modular (#2004)dc458358c0Test util cleanup (#2003)3ca21ebe4dMore strict validation (#2000)a7a7d57331Fix build problem (#1999)fc235b064eJust fixes comments (#1998)482386c050cleanup (#1997)4cbe0db655Improve divisible split detection (#1970)42ccc52bdcMinor build fix. (#1996)fcf8c091f7Cleanup of lower_utils.cpp: Isolate out GpuLower usage (#1989)15f2f6dba8Move ConcretizedBroadcastDomains to shared_ptr in GpuLower. (#1988)8f1c7f5267Minor cleanup lower_unroll.cpp (#1994)1d9858c803Minor cleanup (#1992)f262d9cab5Add support for uniform RNG (#1986)eb1dad10c7Remove non-const functions, remove GpuLower instance on build, pass in ca_map. (#1987)634820c5e3Add support for some empty fusion (#1981)eabe8d844aSegment self mapping fusions (#1954)e96aacfd9cEnable Transpose operation (#1882)425dce2777Add a null scheduler that helps segmenting away no-op schedules (#1835)306d4a68f1Fix canScheduleCompileTime check of transpose scheduler (#1969)b1bd32cc1bMinor fix (#1967)bd93578143Enable transpose scheduler (#1927)b7a206e93bMove scheduler vectorize utilities into their own file (#1959)d9420e4ca0View scheduling (#1928)c668e13aeaUpstream push ci fixes (#1965)c40202bb40Fix dump effective bandwidth (#1962)93505bcbb9WAR on index mapping when exact and permissive maps differ (#1960)45e95fd1d3Allow splitting inner-most ID to create virtual innermost ID in transpose scheduler (#1930)a3ecb33944Improve the comments at the beginning of index_compute.h (#1946)f7bc3417ccRemove unused variables (#1955)df3393adbbSome cleanup (#1957)7d1d7c8724TVDomainGuard factory (#1953)357ba224c0Fill allocation with nan on tests (#1956)8eafc54685Fix detection of unmappable root domains (#1952)90a51f2826Some indexing cleanups, Add eye support (#1940)ddc01e4e16Exclude unsupported data types (#1951)992e17c068test the groups the same order as they are merged (#1949)208262b75dMove detection of self mapping IDs to IterDomainGraph from (#1941)ac4de38c6eMerge pull request #1945 from csarofeen/master_merge_0828631094891aAdd full, full_like, zeros, zeros_like, ones, ones_like (#1943)aab10bce45Merge remote-tracking branch 'upstream/viable/strict' into HEAD4c254c063bFix arange when step is negative (#1942)89330aa23aTensor factories must set the output shape as its input (#1939) ``` RUN_TORCHBENCH: nvfuser Differential Revision: [D40869846](https://our.internmc.facebook.com/intern/diff/D40869846) Pull Request resolved: https://github.com/pytorch/pytorch/pull/87779 Approved by: https://github.com/davidberard98
739 lines
24 KiB
C++
739 lines
24 KiB
C++
#include <torch/csrc/jit/codegen/cuda/executor.h>
|
|
#include <torch/csrc/jit/codegen/cuda/fusion.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_all_nodes.h>
|
|
#include <torch/csrc/jit/codegen/cuda/ir_builder.h>
|
|
|
|
#include <torch/csrc/jit/codegen/cuda/scheduler/all_schedulers.h>
|
|
|
|
#include <benchmark/benchmark.h>
|
|
|
|
#include <benchmarks/cpp/nvfuser/utils.h>
|
|
|
|
using namespace torch::jit::fuser::cuda;
|
|
|
|
static void setup_vit_base_patch16_224_bcast7(Fusion* fusion, void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t2 = makeContigTensor(3, DataType::Float);
|
|
auto t3 = TensorViewBuilder()
|
|
.shape({-1, -1, 1})
|
|
.dtype(DataType::Float)
|
|
.contiguity({true, true, false})
|
|
.build();
|
|
auto t4 = TensorViewBuilder()
|
|
.shape({-1, -1, 1})
|
|
.dtype(DataType::Float)
|
|
.contiguity({true, true, false})
|
|
.build();
|
|
auto t7 = makeContigTensor(3, DataType::Half);
|
|
|
|
fusion->addInput(t2);
|
|
fusion->addInput(t3);
|
|
fusion->addInput(t4);
|
|
fusion->addInput(t7);
|
|
|
|
auto t8 = castOp(DataType::Float, t7);
|
|
auto t9 = set(t8);
|
|
auto t10 = sub(t2, t3);
|
|
auto t11 = mul(t10, t4);
|
|
auto t25 = mul(t9, t11);
|
|
auto t26 = sum(t25, {0, 1});
|
|
auto t36 = set(t26);
|
|
auto t27 = sum(t9, {0, 1});
|
|
auto t37 = set(t27);
|
|
auto t39 = castOp(DataType::Half, t11);
|
|
|
|
fusion->addOutput(t36);
|
|
fusion->addOutput(t37);
|
|
fusion->addOutput(t39);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_bcast7(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t2 = at::randn(input_shape, fp32_options);
|
|
auto t3 = at::randn({input_shape[0], input_shape[1], 1}, fp32_options);
|
|
auto t4 = at::randn({input_shape[0], input_shape[1], 1}, fp32_options);
|
|
auto t7 = at::randn(input_shape, fp16_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t2, t3, t4, t7});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// full tensor - float + halfx2 - t2, t7, t39
|
|
// Inner most dimension only - floatx2 - t36, t37
|
|
// Outer two dimensions only - floatx2 - t3, t4
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
// t2 + t7 t3 + t4 t36 + t37
|
|
t2.numel() * (4 + 2) + t3.numel() * 4 * 2 + input_shape[2] * (4 * 2) +
|
|
// T39
|
|
t2.numel() * 2);
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast7,
|
|
setup_vit_base_patch16_224_bcast7,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast7,
|
|
nullptr);
|
|
|
|
// pwise case, broadcasting both sides
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast7)
|
|
->Args({64, 197, 768})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void setup_vit_base_patch16_224_bcast5(Fusion* fusion, void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t2 = makeContigTensor(3, DataType::Float);
|
|
auto t5 = makeContigTensor(1, DataType::Float);
|
|
auto t3 = makeContigTensor(3, DataType::Half);
|
|
auto t0 = makeContigTensor(1, DataType::Float);
|
|
auto t1 = makeContigTensor(1, DataType::Float);
|
|
|
|
fusion->addInput(t2);
|
|
fusion->addInput(t5);
|
|
fusion->addInput(t3);
|
|
fusion->addInput(t0);
|
|
fusion->addInput(t1);
|
|
|
|
std::vector<bool> bcast_pattern0({true, true, false});
|
|
std::vector<bool> bcast_pattern1({false, false, true});
|
|
|
|
auto t4 = castOp(DataType::Float, t3);
|
|
auto t6 = set(t5);
|
|
auto t7 = broadcast(t6, bcast_pattern0);
|
|
auto t8 = add(t4, t7);
|
|
auto t9 = rand_like(t8);
|
|
auto d34 =
|
|
sub(IrBuilder::create<Double>(1.0), IrBuilder::create<Double>(0.0));
|
|
auto t10 = lt(t9, d34);
|
|
auto t11 = castOp(DataType::Float, t10);
|
|
auto t12 = mul(t8, t11);
|
|
auto b36 = eq(d34, IrBuilder::create<Double>(0.0));
|
|
auto d37 = castOp(DataType::Double, b36);
|
|
auto d38 = add(d37, d34);
|
|
auto d40 = div(IrBuilder::create<Double>(1.0), d38);
|
|
auto t13 = mul(t12, d40);
|
|
auto t14 = set(t13);
|
|
auto t15 = add(t2, t14);
|
|
auto t16 = set(t15);
|
|
auto t36 = sum(t16, {2});
|
|
auto d151 = castOp(DataType::Double, t2->axis(2)->extent());
|
|
auto d152 = mul(IrBuilder::create<Double>(1.0), d151);
|
|
auto t19 = div(t36, d152);
|
|
auto t22 = broadcast(t19, bcast_pattern1);
|
|
auto t23 = sub(t16, t22);
|
|
auto t37 = mul(t23, t23);
|
|
auto t20 = sum(t37, {2});
|
|
auto t24 = broadcast(t20, bcast_pattern1);
|
|
auto d95 = castOp(DataType::Double, t2->axis(2)->extent());
|
|
auto d105 = reciprocal(d95);
|
|
auto t25 = mul(t24, d105);
|
|
auto t26 = add(t25, IrBuilder::create<Double>(1e-6));
|
|
auto t27 = rsqrt(t26);
|
|
auto t28 = mul(t23, t27);
|
|
auto t17 = set(t1);
|
|
auto t29 = broadcast(t17, bcast_pattern0);
|
|
auto t30 = mul(t28, t29);
|
|
auto t18 = set(t0);
|
|
auto t31 = broadcast(t18, bcast_pattern0);
|
|
auto t32 = add(t30, t31);
|
|
auto t33 = set(t32);
|
|
auto t34 = castOp(DataType::Half, t33);
|
|
|
|
fusion->addOutput(t16); // full 3d float
|
|
fusion->addOutput(t10); // full 3d bool
|
|
fusion->addOutput(t22); // bcast last dim float
|
|
fusion->addOutput(t27); // bcast last dim float
|
|
fusion->addOutput(t18); // passthrough t0 float
|
|
fusion->addOutput(t17); // passthrough t1 float
|
|
fusion->addOutput(t34); // full 3d half
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_bcast5(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t2 = at::randn(input_shape, fp32_options);
|
|
auto t5 = at::randn({input_shape[2]}, fp32_options);
|
|
auto t3 = at::randn(input_shape, fp16_options);
|
|
auto t0 = at::randn({input_shape[2]}, fp32_options);
|
|
auto t1 = at::randn({input_shape[2]}, fp32_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t2, t5, t3, t0, t1});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// Full tensor - floatx2, halfx2, bool - t2, t16, t3, t34, t16
|
|
// Inner most dim only - floatx5 - t5, t0, t1, t7, t17
|
|
// Outer two dims only - floatx2 - t22, t27
|
|
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) *
|
|
t2.numel() * (2 * 4 + 2 * 2 + 1) + t5.numel() * 5 * 4 +
|
|
input_shape[0] * input_shape[1] * 2 * 4);
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast5_NCHW,
|
|
setup_vit_base_patch16_224_bcast5,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast5,
|
|
nullptr);
|
|
|
|
// Broadcast on both sides
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_TIMM_vit_base_patch16_224_bcast5_NCHW)
|
|
->Args({64, 197, 768})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void setup_vit_base_patch16_224_bcast_outer2(
|
|
Fusion* fusion,
|
|
void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t0 = makeContigTensor(3, DataType::Half);
|
|
auto t2 = makeContigTensor(1, DataType::Float);
|
|
|
|
fusion->addInput(t0);
|
|
fusion->addInput(t2);
|
|
|
|
auto t1 = castOp(DataType::Float, t0);
|
|
auto t3 = set(t2);
|
|
auto t4 = broadcast(t3, {true, true, false});
|
|
auto t5 = add(t1, t4);
|
|
auto t6 = castOp(DataType::Half, t5);
|
|
auto t7 = castOp(DataType::Half, t3);
|
|
|
|
fusion->addOutput(t6);
|
|
fusion->addOutput(t7);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_outer2(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t0 = at::randn(input_shape, fp16_options);
|
|
auto t2 = at::randn({input_shape[2]}, fp32_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t0, t2});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// full tensor - halfx2 - t0, t6
|
|
// inner dimension only - halfx2 - t2, t7
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * t0.numel() * (2 + 2) +
|
|
input_shape[2] * (2 + 4));
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_outer2,
|
|
setup_vit_base_patch16_224_bcast_outer2,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_outer2,
|
|
nullptr);
|
|
|
|
NVFUSER_BENCHMARK_RUN(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_outer2)
|
|
->Args({64, 197, 2304})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void setup_vit_base_patch16_224_norm_inner3(Fusion* fusion, void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t0 = makeContigTensor(4, DataType::Half);
|
|
fusion->addInput(t0);
|
|
auto d13 = IrBuilder::create<Double>();
|
|
fusion->addInput(d13);
|
|
|
|
auto t1 = castOp(DataType::Float, t0);
|
|
auto t2 = set(t1);
|
|
auto t3 = mul(t2, d13);
|
|
auto t4 = set(t3);
|
|
auto t5 = max(t4, {3});
|
|
auto t6 = broadcast(t5, {false, false, false, true});
|
|
auto t7 = sub(t4, t6);
|
|
auto t8 = exp(t7);
|
|
auto t9 = sum(t8, {3});
|
|
auto t10 = broadcast(t9, {false, false, false, true});
|
|
auto t11 = reciprocal(t10);
|
|
auto t12 = mul(t8, t11);
|
|
auto t13 = rand_like(t12);
|
|
auto d79 = sub(IrBuilder::create<Double>(1), IrBuilder::create<Double>(0));
|
|
auto t14 = lt(t13, d79);
|
|
auto t15 = castOp(DataType::Float, t14);
|
|
auto b81 = eq(d79, IrBuilder::create<Double>(0));
|
|
auto d82 = castOp(DataType::Double, b81);
|
|
auto d83 = add(d82, d79);
|
|
auto d85 = div(IrBuilder::create<Double>(1), d83);
|
|
auto t16 = mul(t12, t15);
|
|
auto t17 = mul(t16, d85);
|
|
auto t18 = set(t17);
|
|
auto t19 = castOp(DataType::Half, t18);
|
|
|
|
fusion->addOutput(t19);
|
|
fusion->addOutput(t14);
|
|
fusion->addOutput(t12);
|
|
fusion->addOutput(t4);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_norm_inner3(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
|
|
auto t0 = at::randn(input_shape, fp16_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t0, 0.125});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// Full tensors - floatx2, half x2, bool - t12, t4, t0, t19, t14
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * t0.numel() * 13);
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_norm_inner3,
|
|
setup_vit_base_patch16_224_norm_inner3,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_norm_inner3,
|
|
nullptr);
|
|
|
|
// Norm inner dim
|
|
NVFUSER_BENCHMARK_RUN(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_norm_inner3)
|
|
->Args({64, 12, 197})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void setup_vit_base_patch16_224_bcast_outer6(
|
|
Fusion* fusion,
|
|
void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t0 = makeContigTensor(3, DataType::Half);
|
|
auto t2 = makeContigTensor(1, DataType::Float);
|
|
|
|
fusion->addInput(t0);
|
|
fusion->addInput(t2);
|
|
|
|
auto t1 = castOp(DataType::Float, t0);
|
|
auto t3 = set(t2);
|
|
auto t4 = broadcast(t3, {true, true, false});
|
|
auto t5 = add(t1, t4);
|
|
auto t6 = set(t5);
|
|
auto t7 = mul(t6, IrBuilder::create<Double>(0.707106));
|
|
auto t8 = erf(t7);
|
|
auto t9 = add(IrBuilder::create<Double>(1), t8);
|
|
auto t10 = mul(IrBuilder::create<Double>(0.5), t9);
|
|
auto t11 = mul(t6, t10);
|
|
auto t12 = rand_like(t11);
|
|
auto d66 = sub(IrBuilder::create<Double>(1), IrBuilder::create<Double>(0));
|
|
auto t13 = lt(t12, d66);
|
|
auto t14 = castOp(DataType::Float, t13);
|
|
auto t15 = mul(t11, t14);
|
|
auto b68 = eq(d66, IrBuilder::create<Double>(0));
|
|
auto d69 = castOp(DataType::Double, b68);
|
|
auto d70 = add(d69, d66);
|
|
auto d72 = div(IrBuilder::create<Double>(1), d70);
|
|
auto t16 = mul(t15, d72);
|
|
auto t17 = set(t16);
|
|
auto t18 = castOp(DataType::Half, t17);
|
|
auto t19 = castOp(DataType::Half, t3);
|
|
|
|
fusion->addOutput(t18);
|
|
fusion->addOutput(t13);
|
|
fusion->addOutput(t6);
|
|
fusion->addOutput(t19);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_outer6(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t0 = at::randn(input_shape, fp16_options);
|
|
auto t2 = at::randn({input_shape[2]}, fp32_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t0, t2});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
// full tensors - float, halfx2, bool - t6, t0, t18, t13
|
|
// inner dimension only - float, half - t2, t19
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * t0.numel() * (2 + 2 + 1 + 4) +
|
|
input_shape[2] * (4 + 2));
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_outer6,
|
|
setup_vit_base_patch16_224_bcast_outer6,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_outer6,
|
|
nullptr);
|
|
|
|
NVFUSER_BENCHMARK_RUN(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_outer6)
|
|
// First size is original, the rest are variations to check perf
|
|
// reliability.
|
|
->Args({64, 197, 3 * 1024})
|
|
->Args({64, 197, 2 * 1024})
|
|
->Args({64, 197, 1024})
|
|
->Args({64, 197, 512})
|
|
->Args({3, 1024, 64 * 197})
|
|
->Args({2, 1024, 64 * 197})
|
|
->Args({1, 1024, 64 * 197})
|
|
->Args({2, 256, 64 * 197})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
// Reverse the broadcast dimensions to check for consistency in scheduling.
|
|
static void setup_vit_base_patch16_224_bcast_inner6(
|
|
Fusion* fusion,
|
|
void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t0 = makeContigTensor(3, DataType::Half);
|
|
auto t2 = makeContigTensor(2, DataType::Float);
|
|
|
|
fusion->addInput(t0);
|
|
fusion->addInput(t2);
|
|
|
|
auto t1 = castOp(DataType::Float, t0);
|
|
auto t3 = set(t2);
|
|
auto t4 = broadcast(t3, {false, false, true});
|
|
auto t5 = add(t1, t4);
|
|
auto t6 = set(t5);
|
|
auto t7 = mul(t6, IrBuilder::create<Double>(0.707106));
|
|
auto t8 = erf(t7);
|
|
auto t9 = add(IrBuilder::create<Double>(1), t8);
|
|
auto t10 = mul(IrBuilder::create<Double>(0.5), t9);
|
|
auto t11 = mul(t6, t10);
|
|
auto t12 = rand_like(t11);
|
|
auto d66 = sub(IrBuilder::create<Double>(1), IrBuilder::create<Double>(0));
|
|
auto t13 = lt(t12, d66);
|
|
auto t14 = castOp(DataType::Float, t13);
|
|
auto t15 = mul(t11, t14);
|
|
auto b68 = eq(d66, IrBuilder::create<Double>(0));
|
|
auto d69 = castOp(DataType::Double, b68);
|
|
auto d70 = add(d69, d66);
|
|
auto d72 = div(IrBuilder::create<Double>(1), d70);
|
|
auto t16 = mul(t15, d72);
|
|
auto t17 = set(t16);
|
|
auto t18 = castOp(DataType::Half, t17);
|
|
auto t19 = castOp(DataType::Half, t3);
|
|
|
|
fusion->addOutput(t18);
|
|
fusion->addOutput(t13);
|
|
fusion->addOutput(t6);
|
|
fusion->addOutput(t19);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_inner6(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t0 = at::randn(input_shape, fp16_options);
|
|
auto t2 = at::randn({input_shape[0], input_shape[1]}, fp32_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t0, t2});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// full tensors - float, halfx2, bool - t6, t0, t18, t13
|
|
// outer two dimensions only - float, half - t2, t19
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * t0.numel() * (2 + 2 + 1 + 4) +
|
|
input_shape[0] * input_shape[1] * (4 + 2));
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_inner6,
|
|
setup_vit_base_patch16_224_bcast_inner6,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_bcast_inner6,
|
|
nullptr);
|
|
|
|
NVFUSER_BENCHMARK_RUN(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_bcast_inner6)
|
|
->Args({64, 197, 3 * 1024})
|
|
->Args({64, 197, 2 * 1024})
|
|
->Args({64, 197, 1024})
|
|
->Args({64, 197, 512})
|
|
->Args({3, 1024, 64 * 197})
|
|
->Args({2, 1024, 64 * 197})
|
|
->Args({1, 1024, 64 * 197})
|
|
->Args({2, 256, 64 * 197})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void setup_vit_base_patch16_224_LN_BWD(Fusion* fusion, void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t0 = makeContigTensor(3, DataType::Bool);
|
|
fusion->addInput(t0);
|
|
|
|
auto t1 = makeContigTensor(3, DataType::Half);
|
|
fusion->addInput(t1);
|
|
|
|
auto t2 = castOp(DataType::Float, t1);
|
|
|
|
auto t3 = makeContigTensor(3, DataType::Half);
|
|
fusion->addInput(t3);
|
|
|
|
auto t4 = castOp(DataType::Float, t3);
|
|
|
|
auto d35 = t3->axis(2)->extent();
|
|
|
|
auto t5 = TensorViewBuilder()
|
|
.shape({-1, -1, 1})
|
|
.dtype(DataType::Float)
|
|
.contiguity({true, true, false})
|
|
.build();
|
|
fusion->addInput(t5);
|
|
|
|
auto t6 = TensorViewBuilder()
|
|
.shape({-1, -1, 1})
|
|
.dtype(DataType::Float)
|
|
.contiguity({true, true, false})
|
|
.build();
|
|
fusion->addInput(t6);
|
|
|
|
auto t7 = makeContigTensor(1, DataType::Half);
|
|
fusion->addInput(t7);
|
|
|
|
auto t8 = castOp(DataType::Float, t7);
|
|
|
|
auto t9 = makeContigTensor(1, DataType::Half);
|
|
fusion->addInput(t9);
|
|
|
|
auto t11 = sub(t4, t5);
|
|
auto t12 = mul(t11, t6);
|
|
|
|
auto t13 = broadcast(t8, {true, true, false});
|
|
auto t14 = mul(t2, t13);
|
|
auto t15 = mul(d35, t14);
|
|
auto t16 = sum(t14, {2});
|
|
auto t17 = broadcast(t16, {false, false, true});
|
|
auto t18 = mul(t14, t12);
|
|
auto t19 = sum(t18, {2});
|
|
auto t20 = broadcast(t19, {false, false, true});
|
|
|
|
auto t40 = castOp(DataType::Half, t12);
|
|
auto t41 = castOp(DataType::Float, t40);
|
|
auto t42 = castOp(DataType::Half, t20);
|
|
auto t43 = castOp(DataType::Float, t42);
|
|
auto t21 = mul(t42, t43);
|
|
|
|
auto t38 = castOp(DataType::Half, t15);
|
|
auto t39 = castOp(DataType::Float, t38);
|
|
auto t44 = castOp(DataType::Half, t17);
|
|
auto t45 = castOp(DataType::Float, t44);
|
|
auto t22 = sub(t39, t45);
|
|
|
|
auto t23 = sub(t22, t21);
|
|
|
|
auto d87 = reciprocal(d35);
|
|
auto t24 = mul(d87, t6);
|
|
|
|
auto t25 = mul(t24, t23);
|
|
auto t26 = mul(t2, t41);
|
|
auto t27 = sum(t26, {0, 1});
|
|
auto t28 = sum(t2, {0, 1});
|
|
|
|
auto t29 = castOp(DataType::Float, t0);
|
|
auto t30 = mul(t25, t29);
|
|
|
|
auto d33 = IrBuilder::create<Double>();
|
|
fusion->addInput(d33);
|
|
auto t31 = mul(t30, d33);
|
|
auto t32 = sum(t31, {0, 1});
|
|
auto t33 = castOp(DataType::Half, t32);
|
|
auto t34 = castOp(DataType::Half, t31);
|
|
auto t35 = castOp(DataType::Half, t25);
|
|
auto t36 = castOp(DataType::Half, t27);
|
|
auto t37 = castOp(DataType::Half, t28);
|
|
|
|
fusion->addOutput(t33);
|
|
fusion->addOutput(t34);
|
|
fusion->addOutput(t35);
|
|
fusion->addOutput(t36);
|
|
fusion->addOutput(t37);
|
|
}
|
|
|
|
static void NvFuserScheduler_TIMM_vit_base_patch16_224_LN_BWD(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(1),
|
|
benchmark_state.range(2)};
|
|
|
|
at::manual_seed(0);
|
|
// auto bool_options = at::TensorOptions().dtype(at::kBool).device(at::kCUDA,
|
|
// 0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
auto fp32_options =
|
|
at::TensorOptions().dtype(at::kFloat).device(at::kCUDA, 0);
|
|
|
|
auto t0 = at::randn(input_shape, fp16_options).to(at::kBool);
|
|
auto t1 = at::randn(input_shape, fp16_options);
|
|
auto t3 = at::randn(input_shape, fp16_options);
|
|
auto t5 = at::randn({input_shape[0], input_shape[1], 1}, fp32_options);
|
|
auto t6 = at::randn({input_shape[0], input_shape[1], 1}, fp32_options);
|
|
auto t7 = at::randn({input_shape[2]}, fp16_options);
|
|
auto t9 = at::randn({input_shape[2]}, fp16_options);
|
|
|
|
std::vector<c10::IValue> aten_inputs({t0, t1, t3, t5, t6, t7, t9, 1.0});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// Full tensors - bool, halfx4 - t0, t1, t3, t34, t35
|
|
// Outer two dimensions - floatx2 - t5, t6
|
|
// Inner dimension - halfx5 - t7, t9, t33, t36, t37
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * ((t0.numel() * (4 * 2 + 1))) +
|
|
(t5.numel() * 4 * 2) + (t7.numel() * 5 * 2));
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_LN_BWD,
|
|
setup_vit_base_patch16_224_LN_BWD,
|
|
NvFuserScheduler_TIMM_vit_base_patch16_224_LN_BWD,
|
|
nullptr);
|
|
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_TIMM_NCHW_vit_base_patch16_224_LN_BWD)
|
|
->Args({128, 197, 768})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|
|
|
|
static void nhwc_seresnet152d_transpose65(Fusion* fusion, void* null) {
|
|
FusionGuard fg(fusion);
|
|
|
|
auto t2 = makeContigTensor(4, DataType::Half);
|
|
auto t5 = makeContigTensor(4, DataType::Half);
|
|
auto t7 = makeContigTensor(4, DataType::Half);
|
|
auto t9 = makeContigTensor(4, DataType::Half);
|
|
auto t4 = makeConcreteTensor({}, DataType::Half);
|
|
|
|
fusion->addInput(t2);
|
|
fusion->addInput(t5);
|
|
fusion->addInput(t7);
|
|
fusion->addInput(t9);
|
|
fusion->addInput(t4);
|
|
|
|
auto d86 = IrBuilder::create<Double>(0);
|
|
|
|
auto t3 = castOp(DataType::Float, t2);
|
|
auto t6 = castOp(DataType::Float, t5);
|
|
auto t8 = castOp(DataType::Float, t7);
|
|
auto t10 = castOp(DataType::Float, t9);
|
|
auto t11 = add(t8, t10);
|
|
auto t12 = set(t11);
|
|
auto t13 = set(t6);
|
|
auto t14 = lt(t13, d86);
|
|
auto t15 = broadcast(t4, {true, true, true, true});
|
|
auto t16 = where(t14, t15, t12);
|
|
auto t17 = set(t16);
|
|
auto t29 = castOp(DataType::Half, t17);
|
|
auto t18 = mul(t17, t3);
|
|
auto t19 = permute(t18, {0, 2, 3, 1});
|
|
auto t30 = castOp(DataType::Half, t19);
|
|
|
|
fusion->addOutput(t29);
|
|
fusion->addOutput(t30);
|
|
}
|
|
|
|
static void NvFuserScheduler_nhwc_seresnet152d_transpose65(
|
|
benchmark::State& benchmark_state,
|
|
FusionExecutorCache* fusion_executor_cache,
|
|
void* null) {
|
|
std::vector<int64_t> input_shape{
|
|
benchmark_state.range(0),
|
|
benchmark_state.range(2),
|
|
benchmark_state.range(2),
|
|
benchmark_state.range(1)};
|
|
|
|
at::manual_seed(0);
|
|
auto fp16_options = at::TensorOptions().dtype(at::kHalf).device(at::kCUDA, 0);
|
|
|
|
auto t2 = at::randn(input_shape, fp16_options);
|
|
auto t5 = at::randn(input_shape, fp16_options);
|
|
auto t7 = at::randn(input_shape, fp16_options);
|
|
auto t9 = at::randn(input_shape, fp16_options);
|
|
// Need zero dim tensor don't know how to do that, so just going to reduce a
|
|
// 1D tensor
|
|
auto t4 = at::randn({2}, fp16_options).sum();
|
|
|
|
std::vector<c10::IValue> aten_inputs({t2, t5, t7, t9, t4});
|
|
runBenchmarkIterations(benchmark_state, fusion_executor_cache, aten_inputs);
|
|
|
|
// Full tensors - halfx6 - t2, t5, t7, t9, t29, t30
|
|
benchmark_state.SetBytesProcessed(
|
|
int64_t(benchmark_state.iterations()) * t2.numel() * 6 * 2);
|
|
}
|
|
|
|
NVFUSER_BENCHMARK_DEFINE(
|
|
NvFuserScheduler_TIMM_nhwc_seresnet152d_transpose65,
|
|
nhwc_seresnet152d_transpose65,
|
|
NvFuserScheduler_nhwc_seresnet152d_transpose65,
|
|
nullptr);
|
|
|
|
// Norm inner dim Half version of vit_base_patch16_224_norm_inner3
|
|
NVFUSER_BENCHMARK_RUN(NvFuserScheduler_TIMM_nhwc_seresnet152d_transpose65)
|
|
->Args({128, 12, 197})
|
|
->Unit(benchmark::kMicrosecond)
|
|
->UseManualTime();
|