mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 08:49:26 +08:00
[c10d] Add an explicit NCCL legacy backend (#191272)
It's incremental work towards switching the default for nccl backend. > AI-assisted description: > > Add an `nccl-legacy` backend entry point that always creates the existing `ProcessGroupNCCL`. Keep `nccl` on that implementation by default, and select the `nccl2` creator when `TORCH_DIST_USE_NCCL2=1`. > > Update NCCL1-specific backend test matrices to use `nccl-legacy` so they continue to test the existing implementation independently of the `nccl` default. > > Test Plan: > > ```bash > BUILD_TEST=0 USE_FBGEMM=0 USE_NNPACK=0 USE_QNNPACK=0 USE_XNNPACK=0 USE_FLASH_ATTENTION=0 USE_MEM_EFF_ATTENTION=0 USE_DISTRIBUTED=1 USE_CUDA=1 MAX_JOBS=144 .venv/bin/pip install -e . -v --no-build-isolation > CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_collectives.py -v > CUDA_VISIBLE_DEVICES=0,1 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_p2p.py -v > CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_cuda_graphs.py -v > CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_process_group.py -v > CUDA_VISIBLE_DEVICES=2,3 NCCL_DEBUG=WARN NCCL_SOCKET_IFNAME=eth0 .venv/bin/python test/distributed/test_c10d_suspend_resume.py -v > .venv/bin/python test/distributed/test_c10d_common.py BackendEntryPointTest -v > spin quicklint > lintrunner -a > ``` > > This change was authored with assistance from an AI coding assistant. Pull Request resolved: https://github.com/pytorch/pytorch/pull/191272 Approved by: https://github.com/kapilsh, https://github.com/fduwjj
This commit is contained in:
committed by
PyTorch MergeBot
parent
fddcbe3411
commit
12a520bc9b
@@ -391,6 +391,7 @@ default = "torch.distributed.elastic.multiprocessing:DefaultLogsSpecs"
|
||||
mpi = "torch.distributed.distributed_c10d:_register_builtin_mpi_backend"
|
||||
gloo = "torch.distributed.distributed_c10d:_register_builtin_gloo_backend"
|
||||
nccl = "torch.distributed.distributed_c10d:_register_builtin_nccl_backend"
|
||||
"nccl-legacy" = "torch.distributed.distributed_c10d:_register_builtin_nccl_legacy_backend"
|
||||
nccl2 = "torch.distributed.distributed_c10d:_register_builtin_nccl2_backend"
|
||||
"nccl-lazy" = "torch.distributed.distributed_c10d:_register_builtin_nccl_lazy_backend"
|
||||
ucc = "torch.distributed.distributed_c10d:_register_builtin_ucc_backend"
|
||||
|
||||
@@ -52,7 +52,7 @@ class BackendConfig:
|
||||
C10D_BACKENDS = (
|
||||
BackendConfig("gloo", "cpu", supports_bitwise_reductions=True),
|
||||
BackendConfig(
|
||||
"nccl",
|
||||
"nccl-legacy",
|
||||
"cuda",
|
||||
supports_coalescing=True,
|
||||
supports_dropped_p2p_work=True,
|
||||
|
||||
@@ -234,6 +234,7 @@ class BackendEntryPointTest(TestCase):
|
||||
)
|
||||
self._custom_backend_attrs = {
|
||||
"ENTRYPOINT_TEST": hasattr(dist.Backend, "ENTRYPOINT_TEST"),
|
||||
"NCCL-LEGACY": hasattr(dist.Backend, "NCCL-LEGACY"),
|
||||
}
|
||||
|
||||
def tearDown(self):
|
||||
@@ -336,6 +337,45 @@ class BackendEntryPointTest(TestCase):
|
||||
self.assertIn("nccl", looked_up)
|
||||
self.assertEqual(str(backend_config), "cpu:gloo,cuda:nccl")
|
||||
|
||||
@parametrize("use_nccl2", [False, True])
|
||||
def test_nccl_backend_registration(self, use_nccl2):
|
||||
with unittest.mock.patch.dict(os.environ):
|
||||
if use_nccl2:
|
||||
os.environ["TORCH_DIST_USE_NCCL2"] = "1"
|
||||
else:
|
||||
os.environ.pop("TORCH_DIST_USE_NCCL2", None)
|
||||
c10d._register_builtin_nccl_backend()
|
||||
|
||||
expected_creator = (
|
||||
c10d._create_nccl2_process_group
|
||||
if use_nccl2
|
||||
else c10d._create_nccl_process_group
|
||||
)
|
||||
self.assertIs(
|
||||
dist.Backend._plugins["NCCL"].creator_fn,
|
||||
expected_creator,
|
||||
)
|
||||
self.assertEqual(
|
||||
dist.Backend.backend_type_map["nccl"],
|
||||
dist.ProcessGroup.BackendType.NCCL,
|
||||
)
|
||||
|
||||
def test_nccl_legacy_backend_registration(self):
|
||||
c10d._register_builtin_nccl_legacy_backend()
|
||||
|
||||
self.assertIs(
|
||||
dist.Backend._plugins["NCCL-LEGACY"].creator_fn,
|
||||
c10d._create_nccl_process_group,
|
||||
)
|
||||
self.assertEqual(dist.Backend.backend_capability["nccl-legacy"], ["cuda"])
|
||||
self.assertEqual(
|
||||
dist.Backend.backend_type_map["nccl-legacy"],
|
||||
dist.ProcessGroup.BackendType.CUSTOM,
|
||||
)
|
||||
|
||||
|
||||
instantiate_parametrized_tests(BackendEntryPointTest)
|
||||
|
||||
|
||||
class Net(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
|
||||
@@ -24,7 +24,7 @@ from torch.testing._internal.common_utils import run_tests, TEST_CUDA
|
||||
|
||||
|
||||
SUSPEND_RESUME_BACKENDS = [
|
||||
("nccl", "cuda"),
|
||||
("nccl-legacy", "cuda"),
|
||||
("nccl2", "cuda"),
|
||||
]
|
||||
|
||||
|
||||
@@ -780,19 +780,31 @@ def _register_builtin_gloo_backend() -> None:
|
||||
|
||||
|
||||
def _register_builtin_nccl_backend() -> None:
|
||||
creator_fn = (
|
||||
_create_nccl2_process_group
|
||||
if os.environ.get("TORCH_DIST_USE_NCCL2") == "1"
|
||||
else _create_nccl_process_group
|
||||
)
|
||||
Backend.register_backend(
|
||||
Backend.NCCL,
|
||||
_create_nccl_process_group,
|
||||
creator_fn,
|
||||
extended_api=True,
|
||||
devices=Backend.backend_capability[Backend.NCCL],
|
||||
_backend_type=ProcessGroup.BackendType.NCCL,
|
||||
)
|
||||
|
||||
|
||||
def _register_builtin_nccl_legacy_backend() -> None:
|
||||
Backend.register_backend(
|
||||
"nccl-legacy",
|
||||
_create_nccl_process_group,
|
||||
extended_api=True,
|
||||
devices=["cuda"],
|
||||
_backend_type=ProcessGroup.BackendType.CUSTOM,
|
||||
)
|
||||
|
||||
|
||||
def _register_builtin_nccl2_backend() -> None:
|
||||
# In-tree torchcomms NCCL backend. CUSTOM backend type; registering with
|
||||
# devices=["cuda"] sets capability without claiming the cuda default (which
|
||||
# stays "nccl"), so this only takes effect when explicitly requested.
|
||||
Backend.register_backend(
|
||||
"nccl2",
|
||||
_create_nccl2_process_group,
|
||||
|
||||
Reference in New Issue
Block a user