mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 08:14:28 +08:00
[CUDA][CI] Bump CUDA 13.0 docker images from 13.0.2 to 13.0.3 to match cuBLAS 13.1.1.3 used by CD (#196869)
## Summary
CI docker images for CUDA 13.0 (`.ci/docker/build.sh`, `install_cuda.sh`) are still pinned to CUDA
toolkit **13.0.2**, but the CD binary build matrix
([`generate_binary_build_matrix.py`](https://github.com/pytorch/pytorch/blob/d1ae7eed74b9ed5e31ac786b6434d2510169135b/.github/scripts/generate_binary_build_matrix.py#L34))
already builds cu130 wheels against **cuBLAS 13.1.1.3**, which ships with CUDA toolkit **13.0.3**, not
13.0.2. This CI/CD toolkit skew means CI tests run the CD-built wheel's cuBLAS 13.1.1.3 against an
older 13.0.2 driver/runtime stack than what it was built/validated for.
This is the root cause of the illegal-memory-access failures in
`TestGroupedMMCUDA.test_grouped_mm_u64_indexing_cuda_float16` on the periodic CUDA 13.0 B200 smoke lane
(e.g. https://github.com/pytorch/pytorch/actions/runs/34476417747/job/102891503623), which only started
showing up after [3c71eadba6](https://github.com/pytorch/pytorch/pull/196075) added coverage exercising this path, and which
#196775 worked around by skipping the test rather than fixing the underlying version mismatch.
- Bump `CUDA_VERSION` 13.0.2 -> 13.0.3 in `.ci/docker/build.sh`, `.ci/docker/common/install_cuda.sh`
(installer bumped to `cuda_13.0.3_580.126.20_linux`, verified to exist at
https://developer.download.nvidia.com/compute/cuda/13.0.3/local_installers/), and
`.ci/docker/common/install_inductor_benchmark_deps.sh` (comment only).
- Same bump for the FA3 wheel build matrix in
`.github/workflows/_binary-build-flash-attention-wheel-linux.yml` (x86_64 + aarch64 legs, including
the aarch64 `cuda_installer_name`).
- `RELEASE.md` supported-architectures table version bump.
- Revert #196775 (`[CUDA] Skip grouped_mm u64 indexing on SM100 CUDA 13.0`) now that the underlying
toolkit/cuBLAS mismatch causing the false failure is fixed.
## Test plan
- [ ] CI docker images rebuild successfully with CUDA 13.0.3.
- [ ] `TestGroupedMMCUDA.test_grouped_mm_u64_indexing_cuda_float16` passes on the periodic CUDA 13.0
B200 smoke lane without the skip from #196775.
- [ ] FA3 wheel build workflow (`_binary-build-flash-attention-wheel-linux.yml`) succeeds for both
x86_64 and aarch64 cu130 legs with the new installer.
Debugged by @nweidia _manually_ but 🤖 Generated with [Claude Code](https://claude.com/claude-code)
p.s. the timing is a bit awkward that we are sunsetting cuda 13.0 CI, so... I think it is a good fyi anyways.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/196869
Approved by: https://github.com/Skylion007
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
committed by
PyTorch MergeBot
parent
7949d70411
commit
f0cb6a19b0
+6
-6
@@ -100,7 +100,7 @@ case "$tag" in
|
|||||||
INSTALL_MINGW=yes
|
INSTALL_MINGW=yes
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11)
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
ANACONDA_PYTHON_VERSION=3.10
|
ANACONDA_PYTHON_VERSION=3.10
|
||||||
GCC_VERSION=11
|
GCC_VERSION=11
|
||||||
KATEX=yes
|
KATEX=yes
|
||||||
@@ -140,7 +140,7 @@ case "$tag" in
|
|||||||
INSTALL_MINGW=yes
|
INSTALL_MINGW=yes
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11)
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
ANACONDA_PYTHON_VERSION=3.12
|
ANACONDA_PYTHON_VERSION=3.12
|
||||||
GCC_VERSION=11
|
GCC_VERSION=11
|
||||||
KATEX=yes
|
KATEX=yes
|
||||||
@@ -148,7 +148,7 @@ case "$tag" in
|
|||||||
INSTALL_MINGW=yes
|
INSTALL_MINGW=yes
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11-inductor-benchmarks)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11-inductor-benchmarks)
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
ANACONDA_PYTHON_VERSION=3.10
|
ANACONDA_PYTHON_VERSION=3.10
|
||||||
GCC_VERSION=11
|
GCC_VERSION=11
|
||||||
KATEX=yes
|
KATEX=yes
|
||||||
@@ -156,7 +156,7 @@ case "$tag" in
|
|||||||
INDUCTOR_BENCHMARKS=yes
|
INDUCTOR_BENCHMARKS=yes
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-inductor-benchmarks)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-inductor-benchmarks)
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
ANACONDA_PYTHON_VERSION=3.12
|
ANACONDA_PYTHON_VERSION=3.12
|
||||||
GCC_VERSION=11
|
GCC_VERSION=11
|
||||||
KATEX=yes
|
KATEX=yes
|
||||||
@@ -164,7 +164,7 @@ case "$tag" in
|
|||||||
INDUCTOR_BENCHMARKS=yes
|
INDUCTOR_BENCHMARKS=yes
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-vllm)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-vllm)
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
ANACONDA_PYTHON_VERSION=3.12
|
ANACONDA_PYTHON_VERSION=3.12
|
||||||
GCC_VERSION=11
|
GCC_VERSION=11
|
||||||
KATEX=yes
|
KATEX=yes
|
||||||
@@ -289,7 +289,7 @@ case "$tag" in
|
|||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-linter)
|
pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-linter)
|
||||||
PYTHON_VERSION=3.10
|
PYTHON_VERSION=3.10
|
||||||
CUDA_VERSION=13.0.2
|
CUDA_VERSION=13.0.3
|
||||||
CLANG_VERSION=18
|
CLANG_VERSION=18
|
||||||
;;
|
;;
|
||||||
pytorch-linux-jammy-aarch64-py3.10-gcc13)
|
pytorch-linux-jammy-aarch64-py3.10-gcc13)
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ function install_130 {
|
|||||||
CUSPARSELT_VERSION=0.8.1.1
|
CUSPARSELT_VERSION=0.8.1.1
|
||||||
echo "Installing CUDA 13.0 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
|
echo "Installing CUDA 13.0 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}"
|
||||||
# install CUDA 13.0 in the same container
|
# install CUDA 13.0 in the same container
|
||||||
install_cuda 13.0.2 cuda_13.0.2_580.95.05_linux
|
install_cuda 13.0.3 cuda_13.0.3_580.126.20_linux
|
||||||
|
|
||||||
# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
|
# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
|
||||||
install_cudnn 13 $CUDNN_VERSION
|
install_cudnn 13 $CUDNN_VERSION
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ sudo apt-get update
|
|||||||
sudo apt-get install -y libpango-1.0-0 libpangocairo-1.0-0
|
sudo apt-get install -y libpango-1.0-0 libpangocairo-1.0-0
|
||||||
|
|
||||||
# Detect CUDA version and use appropriate wheel index
|
# Detect CUDA version and use appropriate wheel index
|
||||||
# DESIRED_CUDA is set as ENV in the Dockerfile (e.g., "13.0.2", "12.8.1")
|
# DESIRED_CUDA is set as ENV in the Dockerfile (e.g., "13.0.3", "12.8.1")
|
||||||
if [[ "${DESIRED_CUDA}" == 13.* ]]; then
|
if [[ "${DESIRED_CUDA}" == 13.* ]]; then
|
||||||
CUDA_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
CUDA_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
||||||
echo "DESIRED_CUDA=${DESIRED_CUDA}, using cu130 wheels"
|
echo "DESIRED_CUDA=${DESIRED_CUDA}, using cu130 wheels"
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ jobs:
|
|||||||
docker_image: "pytorch/manylinux2_28-builder:cuda12.6"
|
docker_image: "pytorch/manylinux2_28-builder:cuda12.6"
|
||||||
runner: "mt-l-x86iavx512-48-384"
|
runner: "mt-l-x86iavx512-48-384"
|
||||||
manylinux_plat: "manylinux_2_28_x86_64"
|
manylinux_plat: "manylinux_2_28_x86_64"
|
||||||
- cuda_version: "13.0.2"
|
- cuda_version: "13.0.3"
|
||||||
arch: "x86_64"
|
arch: "x86_64"
|
||||||
cuda_short: "130"
|
cuda_short: "130"
|
||||||
torch_cuda_arch_list: "8.0;8.6;9.0;10.0"
|
torch_cuda_arch_list: "8.0;8.6;9.0;10.0"
|
||||||
@@ -52,7 +52,7 @@ jobs:
|
|||||||
nvcc_threads: "2"
|
nvcc_threads: "2"
|
||||||
manylinux_plat: "manylinux_2_34_aarch64"
|
manylinux_plat: "manylinux_2_34_aarch64"
|
||||||
cuda_installer_name: "cuda_12.6.3_560.35.05_linux"
|
cuda_installer_name: "cuda_12.6.3_560.35.05_linux"
|
||||||
- cuda_version: "13.0.2"
|
- cuda_version: "13.0.3"
|
||||||
arch: "aarch64"
|
arch: "aarch64"
|
||||||
cuda_short: "130"
|
cuda_short: "130"
|
||||||
torch_cuda_arch_list: "9.0;10.0"
|
torch_cuda_arch_list: "9.0;10.0"
|
||||||
@@ -61,7 +61,7 @@ jobs:
|
|||||||
max_jobs: "4"
|
max_jobs: "4"
|
||||||
nvcc_threads: "2"
|
nvcc_threads: "2"
|
||||||
manylinux_plat: "manylinux_2_34_aarch64"
|
manylinux_plat: "manylinux_2_34_aarch64"
|
||||||
cuda_installer_name: "cuda_13.0.2_580.95.05_linux"
|
cuda_installer_name: "cuda_13.0.3_580.126.20_linux"
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
timeout-minutes: 1440
|
timeout-minutes: 1440
|
||||||
# Build inside the manylinux builder image (the pod is the builder): ARC has
|
# Build inside the manylinux builder image (the pod is the builder): ARC has
|
||||||
|
|||||||
+2
-2
@@ -76,13 +76,13 @@ For Release 2.12, 2.13 and 2.14 PyTorch Supports following CUDA Architectures:
|
|||||||
| CUDA | architectures supported for Linux x86 and Windows builds | notes |
|
| CUDA | architectures supported for Linux x86 and Windows builds | notes |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 12.6.3 | Maxwell(5.0), Pascal(6.0), Volta(7.0), Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0) | |
|
| 12.6.3 | Maxwell(5.0), Pascal(6.0), Volta(7.0), Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0) | |
|
||||||
| 13.0.2 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only |
|
| 13.0.3 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only |
|
||||||
| 13.2.1 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only |
|
| 13.2.1 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only |
|
||||||
|
|
||||||
| CUDA | architectures supported for Linux aarch64 builds |
|
| CUDA | architectures supported for Linux aarch64 builds |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| 12.6.3 | Ampere(8.0), Hopper(9.0) |
|
| 12.6.3 | Ampere(8.0), Hopper(9.0) |
|
||||||
| 13.0.2 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) |
|
| 13.0.3 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) |
|
||||||
| 13.2.1 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) |
|
| 13.2.1 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) |
|
||||||
|
|
||||||
## Release Cadence
|
## Release Cadence
|
||||||
|
|||||||
+1
-5
@@ -42,7 +42,7 @@ from torch.testing._internal.common_dtype import (
|
|||||||
all_types, all_types_and_complex_and, floating_and_complex_types, integral_types,
|
all_types, all_types_and_complex_and, floating_and_complex_types, integral_types,
|
||||||
floating_and_complex_types_and, floating_types_and, complex_types,
|
floating_and_complex_types_and, floating_types_and, complex_types,
|
||||||
)
|
)
|
||||||
from torch.testing._internal.common_cuda import BF16X9_SUPPORTED, CDNA2OrLater, CDNA5OrLater, IS_SM100, SM80OrLater, SM90OrLater, tf32_enabled, tf32_on_and_off, _get_magma_version, \
|
from torch.testing._internal.common_cuda import BF16X9_SUPPORTED, CDNA2OrLater, CDNA5OrLater, SM80OrLater, SM90OrLater, tf32_enabled, tf32_on_and_off, _get_magma_version, \
|
||||||
_get_torch_cuda_version, TEST_MULTIGPU, PLATFORM_SUPPORTS_FP8, blas_library_context, ROCM_VERSION
|
_get_torch_cuda_version, TEST_MULTIGPU, PLATFORM_SUPPORTS_FP8, blas_library_context, ROCM_VERSION
|
||||||
from torch.testing._internal.common_quantization import _group_quantize_tensor, _dynamically_quantize_per_channel, \
|
from torch.testing._internal.common_quantization import _group_quantize_tensor, _dynamically_quantize_per_channel, \
|
||||||
_group_quantize_tensor_symmetric
|
_group_quantize_tensor_symmetric
|
||||||
@@ -11750,10 +11750,6 @@ class TestGroupedMM(TestCase):
|
|||||||
|
|
||||||
@onlyOn(["cuda", "mps"])
|
@onlyOn(["cuda", "mps"])
|
||||||
@skipCUDAIf(not SM80OrLater, "Grouped gemm supported only on SM80 or greater")
|
@skipCUDAIf(not SM80OrLater, "Grouped gemm supported only on SM80 or greater")
|
||||||
@skipCUDAIf(
|
|
||||||
IS_SM100 and _get_torch_cuda_version() == (13, 0),
|
|
||||||
"CUDA 13.0 grouped_mm can cause an illegal memory access on SM100",
|
|
||||||
)
|
|
||||||
@serialTest()
|
@serialTest()
|
||||||
@largeTensorTest("6GB")
|
@largeTensorTest("6GB")
|
||||||
@largeMPSBufferTest((2**31 + 8) * torch.float16.itemsize)
|
@largeMPSBufferTest((2**31 + 8) * torch.float16.itemsize)
|
||||||
|
|||||||
Reference in New Issue
Block a user