From fd34cbee1e25fddee44b4076e6cb967e410c0729 Mon Sep 17 00:00:00 2001 From: Michael Lazos Date: Tue, 1 Sep 2026 20:39:45 -0700 Subject: [PATCH] [inductor][NVGEMM] Generate wider dense EFC candidates (#195539) Generate vendored dense EFC variants for the validated SM100 N-tile widths through 256, including 160, 192, and 224. This makes the configurations available to nvMatmulHeuristics without retaining them unconditionally. Keep the existing supplemented benchmark space unchanged. Wider candidates enter autotuning only when nvMatmulHeuristics selects them, avoiding additional profiling by default. ## Test Plan ```bash PYTHONPATH=/data/users/mlazos/pytorch/agent_space/runtime_overlay PYTORCH_BOUNDARY_ROOT=$PWD python test/inductor/test_nv_universal_gemm.py TestNVUniversalGemm.test_vendored_dense_efc_exposes_wide_n_tiles -v lintrunner -a ``` Authored with an AI assistant. Pull Request resolved: https://github.com/pytorch/pytorch/pull/195539 Approved by: https://github.com/karthickai ghstack dependencies: #192298, #190822, #191514, #191515, #190825, #192928, #195538 --- test/inductor/test_nv_universal_gemm.py | 24 +++++++++++++++++++ .../codegen/nv_universal_gemm/kernel_cache.py | 4 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/test/inductor/test_nv_universal_gemm.py b/test/inductor/test_nv_universal_gemm.py index 286f2e651e2f..56422d9f4a86 100644 --- a/test/inductor/test_nv_universal_gemm.py +++ b/test/inductor/test_nv_universal_gemm.py @@ -371,6 +371,30 @@ class TestNVUniversalGemm(TestCase): self.assertFalse(design.use_2cta_mma) self.assertEqual(design.tile_shape[0], 128) + @unittest.skipIf(IS_FBCODE, "CUTLASS Operator API is not available in fbcode") + def test_vendored_dense_efc_exposes_wide_n_tiles(self): + from cutlass import Float32 + + from torch._inductor.codegen.nv_universal_gemm.kernel_cache import ( + _args_query_candidates, + ) + from torch._inductor.codegen.nv_universal_gemm.nv_universal_gemm_kernel import ( + _create_gemm_arguments, + ) + + m, n, k = 256, 512, 64 + a = torch.empty((m, k), device="cuda", dtype=torch.bfloat16) + b = torch.empty((k, n), device="cuda", dtype=torch.bfloat16) + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + args = _create_gemm_arguments("GEMM", (a, b), out, Float32) + tile_ns = { + kernel.metadata.design.tile_shape[1] + for kernel in _args_query_candidates(args, 100, efc_only=True) + if "VendoredDenseGemmEFCOperator" in kernel.metadata.operator_name + } + + self.assertTrue({64, 128, 160, 192, 224, 256}.issubset(tile_ns)) + def test_efc_epilogue_lookup_no_deadlock(self): """get_efc_kernel_with_epilogue holds _cache_lock and, when the base EFC kernel is not pre-resolved and misses the args cache, calls diff --git a/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py b/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py index 7e0013fef3a3..4c94a08e6d27 100644 --- a/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py +++ b/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py @@ -24,6 +24,8 @@ from torch.utils._ordered_set import OrderedSet log = logging.getLogger(__name__) +DENSE_EFC_TILE_NS = (64, 128, 160, 192, 224, 256) + @functools.cache def _device_target(cc: int) -> Any: @@ -242,7 +244,7 @@ def _replace_dense_efc_with_vendored(kernels: Any) -> list[Any]: continue design = kernel.metadata.design tile_m, tile_n, tile_k = design.tile_shape - for vendored_tile_n in OrderedSet((tile_n, 64, 128)): + for vendored_tile_n in OrderedSet((tile_n, *DENSE_EFC_TILE_NS)): if vendored_tile_n < tile_n: continue operator_name = kernel.metadata.operator_name.replace(