From 156050598eea0bb1124b9c106b22ba4a8c56f82a Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Thu, 3 Sep 2026 22:38:07 -0500 Subject: [PATCH] [ROCm][CI] Bump ROCk base to ROCm 10.0 (#55246) Signed-off-by: Rohan Potdar Co-authored-by: Randall Smith Co-authored-by: Claude Opus 4.8 --- .buildkite/release-pipeline.yaml | 6 +-- docker/Dockerfile.rock_base | 71 ++++++++++---------------------- 2 files changed, 25 insertions(+), 52 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 24da6a14a1..1b444376d6 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -1085,10 +1085,10 @@ steps: agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714" + - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm100" # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm" - - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm" + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm100- vllm/vllm-openai-rocm" + - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm100- vllm/vllm-openai-rocm" plugins: - docker-login#v3.0.0: username: vllmbot diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index d5e507b62d..639ddd01f5 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -1,13 +1,12 @@ ARG BASE_IMAGE=ubuntu:22.04 ARG PYTHON_VERSION=3.12 -ARG TORCH_VERSION=2.12.0+rocm7.14.0 -ARG TORCHVISION_VERSION=0.27.0+rocm7.14.0 -ARG TORCHAUDIO_VERSION=2.11.0+rocm7.14.0 -ARG TRITON_BRANCH="f0b55c07da61c71775bef6d1a15ebf846430ac75" # release/internal/3.7.x as of 08/13 -ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG TORCH_VERSION=2.12.0+rocm10.0.0 +ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0 +ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 +ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0 ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.19" +ARG AITER_BRANCH="v0.1.21.post1" ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 ARG MORI_REPO="https://github.com/ROCm/mori.git" @@ -15,9 +14,9 @@ ARG MORI_REPO="https://github.com/ROCm/mori.git" # TheRock configuration ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages -ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://repo.amd.com/rocm/whl-multi-arch/" # For pre-built GPU images, packages, etc. -ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://rocm.nightlies.amd.com/whl-multi-arch/" # For torch, triton, etc. wheels with latest fixes. -ARG ROCM_SDK_VERSION=7.14.0 +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc. +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes. +ARG ROCM_SDK_VERSION=10.0.0 # Build configuration ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151" @@ -89,16 +88,19 @@ RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ && rocm-sdk init # Install PyTorch wheels (nightly for latest torch, release as fallback) +# Triton is pinned here rather than left to torch's Requires-Dist, so a wheel bump that +# moves the triton torch wants fails resolution instead of quietly changing compilers. ARG TORCH_VERSION ARG TORCHAUDIO_VERSION ARG TORCHVISION_VERSION +ARG TRITON_VERSION RUN pip install \ --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ "torch==${TORCH_VERSION}" \ "torchaudio==${TORCHAUDIO_VERSION}" \ - "torchvision==${TORCHVISION_VERSION}" -RUN python3 -c "import torch; print(f'torch={torch.__version__}, hip={torch.version.hip}')" + "torchvision==${TORCHVISION_VERSION}" \ + "triton==${TRITON_VERSION}" # Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH: @@ -185,20 +187,6 @@ ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} -### -### Triton Build -### -FROM base AS build_triton -ARG TRITON_BRANCH -ARG TRITON_REPO -RUN git clone ${TRITON_REPO} -RUN cd triton \ - && git checkout ${TRITON_BRANCH} \ - && if [ ! -f setup.py ]; then cd python; fi \ - && python3 setup.py bdist_wheel --dist-dir=dist \ - && mkdir -p /app/install && cp dist/*.whl /app/install - - ### ### MORI Build ### @@ -245,8 +233,8 @@ RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO} RUN cd aiter \ && git submodule update --init --recursive \ && pip install -r requirements.txt -# AITER_USE_SYSTEM_TRITON keeps setup.py off the index triton, which would -# otherwise displace the one we built. +# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would +# displace the pinned one already installed in base. RUN pip install pyyaml && cd aiter \ && if [ "$USE_SCCACHE" = "1" ]; then \ export HIP_CLANG_PATH=/opt/sccache-wrappers \ @@ -262,12 +250,10 @@ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install ### Final Build ### -# Wheel release stage - +# Wheel release stage - # only includes dependencies used by wheel release pipeline FROM base AS debs_wheel_release RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ @@ -275,8 +261,6 @@ RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ FROM base AS debs RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ @@ -286,22 +270,15 @@ RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ FROM base AS final -# Remove the torch wheel's Requires-Dist: triton entry in METADATA and the SHA -# hash (its optional) from RECORD. These will cause Python to reinstall -# Triton and a constraints file can't fix it. This doesn't exist for source -# builds, so it wasn't needed there. -RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \ - && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \ - && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \ - && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD" +# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA +# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix. +ENV ROCPROFILER_QUEUE_INTERPOSITION=0 -# Other wheels first, so they resolve against the index triton. Ours goes in last -# with --no-deps, where it can't pull the resolver anywhere. +ARG TRITON_VERSION RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ - pip install $(ls /install/*.whl | grep -v '/triton-') \ - && pip install --no-deps --force-reinstall /install/triton-*.whl \ - # With the pin gone this must be clean, so any complaint is real. + pip install /install/*.whl \ && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \ + && test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \ # Create a constraints file to protect the versions we need, especially # triton, requirements will try to reinstall repeatedly. && pip list --format=freeze \ @@ -321,8 +298,6 @@ ARG TORCHAUDIO_VERSION ARG TORCHVISION_VERSION ARG PYTORCH_ROCM_ARCH ARG AITER_ROCM_ARCH -ARG TRITON_BRANCH -ARG TRITON_REPO ARG FA_BRANCH ARG FA_REPO ARG AITER_BRANCH @@ -338,8 +313,6 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \ && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \ && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \ - && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ - && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \