[Bugfix][ROCm][CI] Stabilize build context and source caches (#51721)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-10 21:51:39 -05:00
committed by GitHub
parent 99e62b802c
commit 419b51b385
2 changed files with 345 additions and 61 deletions
+32 -4
View File
@@ -5,6 +5,7 @@ ARG VLLM_BRANCH="main"
ARG COMMON_WORKDIR=/app
ARG BASE_IMAGE=rocm/vllm-dev:base
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7
# NIC backend for MoRI RDMA support.
# By default (all), drivers and userspace libraries for all supported NIC types
# (ainic and bnxt) are installed; MoRI selects the appropriate one at runtime.
@@ -272,6 +273,20 @@ COPY --from=fetch_vllm \
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
uv pip install --system -r requirements/rocm.txt
# Fetch only the pinned ROCm Triton kernels package. Keeping this before the
# vLLM source COPYs makes the download reusable when native sources change.
FROM base AS rocm-triton-kernels
ARG ROCM_TRITON_KERNELS_COMMIT
ADD https://codeload.github.com/ROCm/triton/tar.gz/${ROCM_TRITON_KERNELS_COMMIT} \
/tmp/rocm-triton.tar.gz
RUN mkdir -p /opt/rocm-triton-kernels \
&& tar --extract --gzip --file /tmp/rocm-triton.tar.gz \
--directory /opt/rocm-triton-kernels --strip-components=4 \
--no-same-owner \
"triton-${ROCM_TRITON_KERNELS_COMMIT}/python/triton_kernels/triton_kernels" \
&& test -f /opt/rocm-triton-kernels/__init__.py \
&& rm /tmp/rocm-triton.tar.gz
# -----------------------
# vLLM native build stages
#
@@ -279,6 +294,11 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
# compilation. That keeps unrelated CI/test/docs edits from invalidating the
# expensive HIP/C++ build layer.
FROM build_vllm_dependencies AS csrc-build
ARG ROCM_TRITON_KERNELS_COMMIT
COPY --from=rocm-triton-kernels \
/opt/rocm-triton-kernels /opt/rocm-triton-kernels
ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
# but it still participates in the BuildKit cache key for this RUN step.
COPY setup.py CMakeLists.txt ./
@@ -292,13 +312,15 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
--mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
export CCACHE_BASEDIR="$PWD" \
grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
cmake/external_projects/triton_kernels.cmake \
&& export CCACHE_BASEDIR="$PWD" \
&& echo "=== ccache stats before ROCm native build ===" \
&& (ccache --show-stats || true) \
&& (ccache --zero-stats || true) \
&& EFFECTIVE_MAX_JOBS="${MAX_JOBS:-$(nproc)}" \
&& echo "Building ROCm native extension wheel with MAX_JOBS=${EFFECTIVE_MAX_JOBS}" \
&& LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \
&& PYTHONDONTWRITEBYTECODE=1 LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \
&& test -d dist \
&& ls dist/*.whl >/dev/null \
&& echo "=== ccache stats after ROCm native build ===" \
@@ -322,7 +344,7 @@ RUN cd "${COMMON_WORKDIR}/vllm" \
&& export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
&& export VLLM_DOCKER_BUILD_CONTEXT=1 \
&& echo "Packaging vLLM ROCm wheel using precompiled extensions from ${VLLM_PRECOMPILED_WHEEL_LOCATION}" \
&& python3 setup.py bdist_wheel --dist-dir=dist \
&& PYTHONDONTWRITEBYTECODE=1 python3 setup.py bdist_wheel --dist-dir=dist \
&& test -d dist \
&& ls dist/*.whl >/dev/null
FROM scratch AS export_vllm
@@ -551,6 +573,10 @@ RUN /bin/bash -lc 'set -euo pipefail; \
FROM fetch_vllm AS build_vllm_wheel_release
ARG COMMON_WORKDIR
ARG ROCM_TRITON_KERNELS_COMMIT
COPY --from=rocm-triton-kernels \
/opt/rocm-triton-kernels /opt/rocm-triton-kernels
ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels
# Drop the pre-built Rust artifacts into the source tree. setup.py detects
# them and ships them as-is, skipping the local Rust build.
@@ -590,7 +616,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \
# This ensures setuptools_scm sees clean repo state for version detection
RUN --mount=type=bind,source=.git,target=vllm/.git \
--mount=type=cache,target=/root/.cache/uv \
cd vllm \
grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
vllm/cmake/external_projects/triton_kernels.cmake \
&& cd vllm \
&& uv pip install --system setuptools_scm regex \
&& VLLM_VERSION=$(python3 -c "import setuptools_scm; print(setuptools_scm.get_version())") \
&& echo "Detected vLLM version: ${VLLM_VERSION}" \