[ROCm][CI] Bump ROCk base to ROCm 10.0 (#55246)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Randall Smith <Randall.Smith@amd.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Randall Smith
Claude Opus 4.8
parent
8a0a7ee40a
commit
156050598e
@@ -1085,10 +1085,10 @@ steps:
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714"
|
||||
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm100"
|
||||
# Clean up old nightly builds (keep only last 14)
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm"
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm"
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm100- vllm/vllm-openai-rocm"
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm100- vllm/vllm-openai-rocm"
|
||||
plugins:
|
||||
- docker-login#v3.0.0:
|
||||
username: vllmbot
|
||||
|
||||
+22
-49
@@ -1,13 +1,12 @@
|
||||
ARG BASE_IMAGE=ubuntu:22.04
|
||||
ARG PYTHON_VERSION=3.12
|
||||
ARG TORCH_VERSION=2.12.0+rocm7.14.0
|
||||
ARG TORCHVISION_VERSION=0.27.0+rocm7.14.0
|
||||
ARG TORCHAUDIO_VERSION=2.11.0+rocm7.14.0
|
||||
ARG TRITON_BRANCH="f0b55c07da61c71775bef6d1a15ebf846430ac75" # release/internal/3.7.x as of 08/13
|
||||
ARG TRITON_REPO="https://github.com/ROCm/triton.git"
|
||||
ARG TORCH_VERSION=2.12.0+rocm10.0.0
|
||||
ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0
|
||||
ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0
|
||||
ARG TRITON_VERSION=3.8.0+git4cff872c.rocm10.0.0
|
||||
ARG FA_BRANCH="0e60e394"
|
||||
ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
|
||||
ARG AITER_BRANCH="v0.1.19"
|
||||
ARG AITER_BRANCH="v0.1.21.post1"
|
||||
ARG AITER_REPO="https://github.com/ROCm/aiter.git"
|
||||
ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0
|
||||
ARG MORI_REPO="https://github.com/ROCm/mori.git"
|
||||
@@ -15,9 +14,9 @@ ARG MORI_REPO="https://github.com/ROCm/mori.git"
|
||||
|
||||
# TheRock configuration
|
||||
ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages
|
||||
ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://repo.amd.com/rocm/whl-multi-arch/" # For pre-built GPU images, packages, etc.
|
||||
ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://rocm.nightlies.amd.com/whl-multi-arch/" # For torch, triton, etc. wheels with latest fixes.
|
||||
ARG ROCM_SDK_VERSION=7.14.0
|
||||
ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc.
|
||||
ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes.
|
||||
ARG ROCM_SDK_VERSION=10.0.0
|
||||
|
||||
# Build configuration
|
||||
ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151"
|
||||
@@ -89,16 +88,19 @@ RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
|
||||
&& rocm-sdk init
|
||||
|
||||
# Install PyTorch wheels (nightly for latest torch, release as fallback)
|
||||
# Triton is pinned here rather than left to torch's Requires-Dist, so a wheel bump that
|
||||
# moves the triton torch wants fails resolution instead of quietly changing compilers.
|
||||
ARG TORCH_VERSION
|
||||
ARG TORCHAUDIO_VERSION
|
||||
ARG TORCHVISION_VERSION
|
||||
ARG TRITON_VERSION
|
||||
RUN pip install \
|
||||
--index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
|
||||
--extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
|
||||
"torch==${TORCH_VERSION}" \
|
||||
"torchaudio==${TORCHAUDIO_VERSION}" \
|
||||
"torchvision==${TORCHVISION_VERSION}"
|
||||
RUN python3 -c "import torch; print(f'torch={torch.__version__}, hip={torch.version.hip}')"
|
||||
"torchvision==${TORCHVISION_VERSION}" \
|
||||
"triton==${TRITON_VERSION}"
|
||||
|
||||
# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH:
|
||||
|
||||
@@ -185,20 +187,6 @@ ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
|
||||
ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
|
||||
|
||||
|
||||
###
|
||||
### Triton Build
|
||||
###
|
||||
FROM base AS build_triton
|
||||
ARG TRITON_BRANCH
|
||||
ARG TRITON_REPO
|
||||
RUN git clone ${TRITON_REPO}
|
||||
RUN cd triton \
|
||||
&& git checkout ${TRITON_BRANCH} \
|
||||
&& if [ ! -f setup.py ]; then cd python; fi \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& mkdir -p /app/install && cp dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
### MORI Build
|
||||
###
|
||||
@@ -245,8 +233,8 @@ RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO}
|
||||
RUN cd aiter \
|
||||
&& git submodule update --init --recursive \
|
||||
&& pip install -r requirements.txt
|
||||
# AITER_USE_SYSTEM_TRITON keeps setup.py off the index triton, which would
|
||||
# otherwise displace the one we built.
|
||||
# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would
|
||||
# displace the pinned one already installed in base.
|
||||
RUN pip install pyyaml && cd aiter \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
export HIP_CLANG_PATH=/opt/sccache-wrappers \
|
||||
@@ -262,12 +250,10 @@ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
|
||||
### Final Build
|
||||
###
|
||||
|
||||
# Wheel release stage -
|
||||
# Wheel release stage -
|
||||
# only includes dependencies used by wheel release pipeline
|
||||
FROM base AS debs_wheel_release
|
||||
RUN mkdir /app/debs
|
||||
RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
@@ -275,8 +261,6 @@ RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
|
||||
FROM base AS debs
|
||||
RUN mkdir /app/debs
|
||||
RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
@@ -286,22 +270,15 @@ RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
|
||||
|
||||
FROM base AS final
|
||||
|
||||
# Remove the torch wheel's Requires-Dist: triton entry in METADATA and the SHA
|
||||
# hash (its optional) from RECORD. These will cause Python to reinstall
|
||||
# Triton and a constraints file can't fix it. This doesn't exist for source
|
||||
# builds, so it wasn't needed there.
|
||||
RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \
|
||||
&& grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \
|
||||
&& sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \
|
||||
&& sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD"
|
||||
# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA
|
||||
# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix.
|
||||
ENV ROCPROFILER_QUEUE_INTERPOSITION=0
|
||||
|
||||
# Other wheels first, so they resolve against the index triton. Ours goes in last
|
||||
# with --no-deps, where it can't pull the resolver anywhere.
|
||||
ARG TRITON_VERSION
|
||||
RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
|
||||
pip install $(ls /install/*.whl | grep -v '/triton-') \
|
||||
&& pip install --no-deps --force-reinstall /install/triton-*.whl \
|
||||
# With the pin gone this must be clean, so any complaint is real.
|
||||
pip install /install/*.whl \
|
||||
&& ! pip check 2>&1 | grep -iE '^(torch|triton) ' \
|
||||
&& test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \
|
||||
# Create a constraints file to protect the versions we need, especially
|
||||
# triton, requirements will try to reinstall repeatedly.
|
||||
&& pip list --format=freeze \
|
||||
@@ -321,8 +298,6 @@ ARG TORCHAUDIO_VERSION
|
||||
ARG TORCHVISION_VERSION
|
||||
ARG PYTORCH_ROCM_ARCH
|
||||
ARG AITER_ROCM_ARCH
|
||||
ARG TRITON_BRANCH
|
||||
ARG TRITON_REPO
|
||||
ARG FA_BRANCH
|
||||
ARG FA_REPO
|
||||
ARG AITER_BRANCH
|
||||
@@ -338,8 +313,6 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
|
||||
&& echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \
|
||||
&& echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \
|
||||
&& echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \
|
||||
&& echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
|
||||
&& echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
|
||||
&& echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
|
||||
|
||||
Reference in New Issue
Block a user