[c10d] Add an explicit NCCL legacy backend (#191272)

It's incremental work towards switching the default for nccl backend.

> AI-assisted description:
>
> Add an `nccl-legacy` backend entry point that always creates the existing `ProcessGroupNCCL`. Keep `nccl` on that implementation by default, and select the `nccl2` creator when `TORCH_DIST_USE_NCCL2=1`.
>
> Update NCCL1-specific backend test matrices to use `nccl-legacy` so they continue to test the existing implementation independently of the `nccl` default.
>
> Test Plan:
>
> ```bash
> BUILD_TEST=0 USE_FBGEMM=0 USE_NNPACK=0 USE_QNNPACK=0 USE_XNNPACK=0 USE_FLASH_ATTENTION=0 USE_MEM_EFF_ATTENTION=0 USE_DISTRIBUTED=1 USE_CUDA=1 MAX_JOBS=144 .venv/bin/pip install -e . -v --no-build-isolation
> CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_collectives.py -v
> CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_p2p.py -v
> CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_cuda_graphs.py -v
> CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_process_group.py -v
> CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_suspend_resume.py -v
> .venv/bin/python test/distributed/test_c10d_common.py BackendEntryPointTest -v
> spin quicklint
> lintrunner -a
> ```
>
> This change was authored with assistance from an AI coding assistant.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/191272
Approved by: https://github.com/kapilsh, https://github.com/fduwjj
This commit is contained in:
Tristan Rice
2026-07-27 23:08:40 +00:00
committed by PyTorch MergeBot
parent fddcbe3411
commit 12a520bc9b
5 changed files with 59 additions and 6 deletions
+1
View File
@@ -391,6 +391,7 @@ default = "torch.distributed.elastic.multiprocessing:DefaultLogsSpecs"
mpi = "torch.distributed.distributed_c10d:_register_builtin_mpi_backend"
gloo = "torch.distributed.distributed_c10d:_register_builtin_gloo_backend"
nccl = "torch.distributed.distributed_c10d:_register_builtin_nccl_backend"
"nccl-legacy" = "torch.distributed.distributed_c10d:_register_builtin_nccl_legacy_backend"
nccl2 = "torch.distributed.distributed_c10d:_register_builtin_nccl2_backend"
"nccl-lazy" = "torch.distributed.distributed_c10d:_register_builtin_nccl_lazy_backend"
ucc = "torch.distributed.distributed_c10d:_register_builtin_ucc_backend"
+1 -1
View File
@@ -52,7 +52,7 @@ class BackendConfig:
C10D_BACKENDS = (
BackendConfig("gloo", "cpu", supports_bitwise_reductions=True),
BackendConfig(
"nccl",
"nccl-legacy",
"cuda",
supports_coalescing=True,
supports_dropped_p2p_work=True,
+40
View File
@@ -234,6 +234,7 @@ class BackendEntryPointTest(TestCase):
)
self._custom_backend_attrs = {
"ENTRYPOINT_TEST": hasattr(dist.Backend, "ENTRYPOINT_TEST"),
"NCCL-LEGACY": hasattr(dist.Backend, "NCCL-LEGACY"),
}
def tearDown(self):
@@ -336,6 +337,45 @@ class BackendEntryPointTest(TestCase):
self.assertIn("nccl", looked_up)
self.assertEqual(str(backend_config), "cpu:gloo,cuda:nccl")
@parametrize("use_nccl2", [False, True])
def test_nccl_backend_registration(self, use_nccl2):
with unittest.mock.patch.dict(os.environ):
if use_nccl2:
os.environ["TORCH_DIST_USE_NCCL2"] = "1"
else:
os.environ.pop("TORCH_DIST_USE_NCCL2", None)
c10d._register_builtin_nccl_backend()
expected_creator = (
c10d._create_nccl2_process_group
if use_nccl2
else c10d._create_nccl_process_group
)
self.assertIs(
dist.Backend._plugins["NCCL"].creator_fn,
expected_creator,
)
self.assertEqual(
dist.Backend.backend_type_map["nccl"],
dist.ProcessGroup.BackendType.NCCL,
)
def test_nccl_legacy_backend_registration(self):
c10d._register_builtin_nccl_legacy_backend()
self.assertIs(
dist.Backend._plugins["NCCL-LEGACY"].creator_fn,
c10d._create_nccl_process_group,
)
self.assertEqual(dist.Backend.backend_capability["nccl-legacy"], ["cuda"])
self.assertEqual(
dist.Backend.backend_type_map["nccl-legacy"],
dist.ProcessGroup.BackendType.CUSTOM,
)
instantiate_parametrized_tests(BackendEntryPointTest)
class Net(nn.Module):
def __init__(self) -> None:
+1 -1
View File
@@ -24,7 +24,7 @@ from torch.testing._internal.common_utils import run_tests, TEST_CUDA
SUSPEND_RESUME_BACKENDS = [
("nccl", "cuda"),
("nccl-legacy", "cuda"),
("nccl2", "cuda"),
]
+16 -4
View File
@@ -780,19 +780,31 @@ def _register_builtin_gloo_backend() -> None:
def _register_builtin_nccl_backend() -> None:
creator_fn = (
_create_nccl2_process_group
if os.environ.get("TORCH_DIST_USE_NCCL2") == "1"
else _create_nccl_process_group
)
Backend.register_backend(
Backend.NCCL,
_create_nccl_process_group,
creator_fn,
extended_api=True,
devices=Backend.backend_capability[Backend.NCCL],
_backend_type=ProcessGroup.BackendType.NCCL,
)
def _register_builtin_nccl_legacy_backend() -> None:
Backend.register_backend(
"nccl-legacy",
_create_nccl_process_group,
extended_api=True,
devices=["cuda"],
_backend_type=ProcessGroup.BackendType.CUSTOM,
)
def _register_builtin_nccl2_backend() -> None:
# In-tree torchcomms NCCL backend. CUSTOM backend type; registering with
# devices=["cuda"] sets capability without claiming the cuda default (which
# stays "nccl"), so this only takes effect when explicitly requested.
Backend.register_backend(
"nccl2",
_create_nccl2_process_group,