diff --git a/test/inductor/test_nv_universal_gemm.py b/test/inductor/test_nv_universal_gemm.py index 286f2e651e2f..56422d9f4a86 100644 --- a/test/inductor/test_nv_universal_gemm.py +++ b/test/inductor/test_nv_universal_gemm.py @@ -371,6 +371,30 @@ class TestNVUniversalGemm(TestCase): self.assertFalse(design.use_2cta_mma) self.assertEqual(design.tile_shape[0], 128) + @unittest.skipIf(IS_FBCODE, "CUTLASS Operator API is not available in fbcode") + def test_vendored_dense_efc_exposes_wide_n_tiles(self): + from cutlass import Float32 + + from torch._inductor.codegen.nv_universal_gemm.kernel_cache import ( + _args_query_candidates, + ) + from torch._inductor.codegen.nv_universal_gemm.nv_universal_gemm_kernel import ( + _create_gemm_arguments, + ) + + m, n, k = 256, 512, 64 + a = torch.empty((m, k), device="cuda", dtype=torch.bfloat16) + b = torch.empty((k, n), device="cuda", dtype=torch.bfloat16) + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + args = _create_gemm_arguments("GEMM", (a, b), out, Float32) + tile_ns = { + kernel.metadata.design.tile_shape[1] + for kernel in _args_query_candidates(args, 100, efc_only=True) + if "VendoredDenseGemmEFCOperator" in kernel.metadata.operator_name + } + + self.assertTrue({64, 128, 160, 192, 224, 256}.issubset(tile_ns)) + def test_efc_epilogue_lookup_no_deadlock(self): """get_efc_kernel_with_epilogue holds _cache_lock and, when the base EFC kernel is not pre-resolved and misses the args cache, calls diff --git a/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py b/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py index 7e0013fef3a3..4c94a08e6d27 100644 --- a/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py +++ b/torch/_inductor/codegen/nv_universal_gemm/kernel_cache.py @@ -24,6 +24,8 @@ from torch.utils._ordered_set import OrderedSet log = logging.getLogger(__name__) +DENSE_EFC_TILE_NS = (64, 128, 160, 192, 224, 256) + @functools.cache def _device_target(cc: int) -> Any: @@ -242,7 +244,7 @@ def _replace_dense_efc_with_vendored(kernels: Any) -> list[Any]: continue design = kernel.metadata.design tile_m, tile_n, tile_k = design.tile_shape - for vendored_tile_n in OrderedSet((tile_n, 64, 128)): + for vendored_tile_n in OrderedSet((tile_n, *DENSE_EFC_TILE_NS)): if vendored_tile_n < tile_n: continue operator_name = kernel.metadata.operator_name.replace(