From f0cb6a19b09276b66ada198a6b5637265ad24744 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Sat, 12 Sep 2026 22:59:32 +0000 Subject: [PATCH] [CUDA][CI] Bump CUDA 13.0 docker images from 13.0.2 to 13.0.3 to match cuBLAS 13.1.1.3 used by CD (#196869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary CI docker images for CUDA 13.0 (`.ci/docker/build.sh`, `install_cuda.sh`) are still pinned to CUDA toolkit **13.0.2**, but the CD binary build matrix ([`generate_binary_build_matrix.py`](https://github.com/pytorch/pytorch/blob/d1ae7eed74b9ed5e31ac786b6434d2510169135b/.github/scripts/generate_binary_build_matrix.py#L34)) already builds cu130 wheels against **cuBLAS 13.1.1.3**, which ships with CUDA toolkit **13.0.3**, not 13.0.2. This CI/CD toolkit skew means CI tests run the CD-built wheel's cuBLAS 13.1.1.3 against an older 13.0.2 driver/runtime stack than what it was built/validated for. This is the root cause of the illegal-memory-access failures in `TestGroupedMMCUDA.test_grouped_mm_u64_indexing_cuda_float16` on the periodic CUDA 13.0 B200 smoke lane (e.g. https://github.com/pytorch/pytorch/actions/runs/34476417747/job/102891503623), which only started showing up after [3c71eadba6ffd0d19c4524e723afd4cba966517a](https://github.com/pytorch/pytorch/pull/196075) added coverage exercising this path, and which #196775 worked around by skipping the test rather than fixing the underlying version mismatch. - Bump `CUDA_VERSION` 13.0.2 -> 13.0.3 in `.ci/docker/build.sh`, `.ci/docker/common/install_cuda.sh` (installer bumped to `cuda_13.0.3_580.126.20_linux`, verified to exist at https://developer.download.nvidia.com/compute/cuda/13.0.3/local_installers/), and `.ci/docker/common/install_inductor_benchmark_deps.sh` (comment only). - Same bump for the FA3 wheel build matrix in `.github/workflows/_binary-build-flash-attention-wheel-linux.yml` (x86_64 + aarch64 legs, including the aarch64 `cuda_installer_name`). - `RELEASE.md` supported-architectures table version bump. - Revert #196775 (`[CUDA] Skip grouped_mm u64 indexing on SM100 CUDA 13.0`) now that the underlying toolkit/cuBLAS mismatch causing the false failure is fixed. ## Test plan - [ ] CI docker images rebuild successfully with CUDA 13.0.3. - [ ] `TestGroupedMMCUDA.test_grouped_mm_u64_indexing_cuda_float16` passes on the periodic CUDA 13.0 B200 smoke lane without the skip from #196775. - [ ] FA3 wheel build workflow (`_binary-build-flash-attention-wheel-linux.yml`) succeeds for both x86_64 and aarch64 cu130 legs with the new installer. Debugged by @nweidia _manually_ but 🤖 Generated with [Claude Code](https://claude.com/claude-code) p.s. the timing is a bit awkward that we are sunsetting cuda 13.0 CI, so... I think it is a good fyi anyways. Pull Request resolved: https://github.com/pytorch/pytorch/pull/196869 Approved by: https://github.com/Skylion007 Co-authored-by: Claude Sonnet 5 --- .ci/docker/build.sh | 12 ++++++------ .ci/docker/common/install_cuda.sh | 2 +- .ci/docker/common/install_inductor_benchmark_deps.sh | 2 +- .../_binary-build-flash-attention-wheel-linux.yml | 6 +++--- RELEASE.md | 4 ++-- test/test_linalg.py | 6 +----- 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index d6bf09df1c00..e62aa74127f5 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -100,7 +100,7 @@ case "$tag" in INSTALL_MINGW=yes ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11) - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 ANACONDA_PYTHON_VERSION=3.10 GCC_VERSION=11 KATEX=yes @@ -140,7 +140,7 @@ case "$tag" in INSTALL_MINGW=yes ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11) - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 ANACONDA_PYTHON_VERSION=3.12 GCC_VERSION=11 KATEX=yes @@ -148,7 +148,7 @@ case "$tag" in INSTALL_MINGW=yes ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3-gcc11-inductor-benchmarks) - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 ANACONDA_PYTHON_VERSION=3.10 GCC_VERSION=11 KATEX=yes @@ -156,7 +156,7 @@ case "$tag" in INDUCTOR_BENCHMARKS=yes ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-inductor-benchmarks) - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 ANACONDA_PYTHON_VERSION=3.12 GCC_VERSION=11 KATEX=yes @@ -164,7 +164,7 @@ case "$tag" in INDUCTOR_BENCHMARKS=yes ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3.12-gcc11-vllm) - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 ANACONDA_PYTHON_VERSION=3.12 GCC_VERSION=11 KATEX=yes @@ -289,7 +289,7 @@ case "$tag" in ;; pytorch-linux-jammy-cuda13.0-cudnn9-py3.10-linter) PYTHON_VERSION=3.10 - CUDA_VERSION=13.0.2 + CUDA_VERSION=13.0.3 CLANG_VERSION=18 ;; pytorch-linux-jammy-aarch64-py3.10-gcc13) diff --git a/.ci/docker/common/install_cuda.sh b/.ci/docker/common/install_cuda.sh index 9963a5a98091..7a2f1fa60088 100644 --- a/.ci/docker/common/install_cuda.sh +++ b/.ci/docker/common/install_cuda.sh @@ -167,7 +167,7 @@ function install_130 { CUSPARSELT_VERSION=0.8.1.1 echo "Installing CUDA 13.0 and cuDNN ${CUDNN_VERSION} and NVSHMEM and NCCL and cuSparseLt-${CUSPARSELT_VERSION}" # install CUDA 13.0 in the same container - install_cuda 13.0.2 cuda_13.0.2_580.95.05_linux + install_cuda 13.0.3 cuda_13.0.3_580.126.20_linux # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement install_cudnn 13 $CUDNN_VERSION diff --git a/.ci/docker/common/install_inductor_benchmark_deps.sh b/.ci/docker/common/install_inductor_benchmark_deps.sh index 2850fe0971ff..29473564017e 100644 --- a/.ci/docker/common/install_inductor_benchmark_deps.sh +++ b/.ci/docker/common/install_inductor_benchmark_deps.sh @@ -35,7 +35,7 @@ sudo apt-get update sudo apt-get install -y libpango-1.0-0 libpangocairo-1.0-0 # Detect CUDA version and use appropriate wheel index -# DESIRED_CUDA is set as ENV in the Dockerfile (e.g., "13.0.2", "12.8.1") +# DESIRED_CUDA is set as ENV in the Dockerfile (e.g., "13.0.3", "12.8.1") if [[ "${DESIRED_CUDA}" == 13.* ]]; then CUDA_INDEX_URL="https://download.pytorch.org/whl/cu130" echo "DESIRED_CUDA=${DESIRED_CUDA}, using cu130 wheels" diff --git a/.github/workflows/_binary-build-flash-attention-wheel-linux.yml b/.github/workflows/_binary-build-flash-attention-wheel-linux.yml index dacd801a88f8..a75cd411834b 100644 --- a/.github/workflows/_binary-build-flash-attention-wheel-linux.yml +++ b/.github/workflows/_binary-build-flash-attention-wheel-linux.yml @@ -35,7 +35,7 @@ jobs: docker_image: "pytorch/manylinux2_28-builder:cuda12.6" runner: "mt-l-x86iavx512-48-384" manylinux_plat: "manylinux_2_28_x86_64" - - cuda_version: "13.0.2" + - cuda_version: "13.0.3" arch: "x86_64" cuda_short: "130" torch_cuda_arch_list: "8.0;8.6;9.0;10.0" @@ -52,7 +52,7 @@ jobs: nvcc_threads: "2" manylinux_plat: "manylinux_2_34_aarch64" cuda_installer_name: "cuda_12.6.3_560.35.05_linux" - - cuda_version: "13.0.2" + - cuda_version: "13.0.3" arch: "aarch64" cuda_short: "130" torch_cuda_arch_list: "9.0;10.0" @@ -61,7 +61,7 @@ jobs: max_jobs: "4" nvcc_threads: "2" manylinux_plat: "manylinux_2_34_aarch64" - cuda_installer_name: "cuda_13.0.2_580.95.05_linux" + cuda_installer_name: "cuda_13.0.3_580.126.20_linux" runs-on: ${{ matrix.runner }} timeout-minutes: 1440 # Build inside the manylinux builder image (the pod is the builder): ARC has diff --git a/RELEASE.md b/RELEASE.md index 35316fe0e5a3..7a77180f75be 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -76,13 +76,13 @@ For Release 2.12, 2.13 and 2.14 PyTorch Supports following CUDA Architectures: | CUDA | architectures supported for Linux x86 and Windows builds | notes | | --- | --- | --- | | 12.6.3 | Maxwell(5.0), Pascal(6.0), Volta(7.0), Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0) | | -| 13.0.2 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only | +| 13.0.3 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only | | 13.2.1 | Turing(7.5), Ampere(8.0, 8.6), Hopper(9.0), Blackwell(10.0, 12.0+PTX) | +PTX available on linux builds only | | CUDA | architectures supported for Linux aarch64 builds | | --- | --- | | 12.6.3 | Ampere(8.0), Hopper(9.0) | -| 13.0.2 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) | +| 13.0.3 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) | | 13.2.1 | Ampere(8.0), Hopper(9.0), Blackwell(10.0, 11.0, 12.0+PTX) | ## Release Cadence diff --git a/test/test_linalg.py b/test/test_linalg.py index 3153f1741f38..365b3d329457 100644 --- a/test/test_linalg.py +++ b/test/test_linalg.py @@ -42,7 +42,7 @@ from torch.testing._internal.common_dtype import ( all_types, all_types_and_complex_and, floating_and_complex_types, integral_types, floating_and_complex_types_and, floating_types_and, complex_types, ) -from torch.testing._internal.common_cuda import BF16X9_SUPPORTED, CDNA2OrLater, CDNA5OrLater, IS_SM100, SM80OrLater, SM90OrLater, tf32_enabled, tf32_on_and_off, _get_magma_version, \ +from torch.testing._internal.common_cuda import BF16X9_SUPPORTED, CDNA2OrLater, CDNA5OrLater, SM80OrLater, SM90OrLater, tf32_enabled, tf32_on_and_off, _get_magma_version, \ _get_torch_cuda_version, TEST_MULTIGPU, PLATFORM_SUPPORTS_FP8, blas_library_context, ROCM_VERSION from torch.testing._internal.common_quantization import _group_quantize_tensor, _dynamically_quantize_per_channel, \ _group_quantize_tensor_symmetric @@ -11750,10 +11750,6 @@ class TestGroupedMM(TestCase): @onlyOn(["cuda", "mps"]) @skipCUDAIf(not SM80OrLater, "Grouped gemm supported only on SM80 or greater") - @skipCUDAIf( - IS_SM100 and _get_torch_cuda_version() == (13, 0), - "CUDA 13.0 grouped_mm can cause an illegal memory access on SM100", - ) @serialTest() @largeTensorTest("6GB") @largeMPSBufferTest((2**31 + 8) * torch.float16.itemsize)