[ROCm][CI] Reuse equivalent ROCm CI images (#48646)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
f18e10a7e1
commit
7f6432cc0d
+25
-18
@@ -253,21 +253,32 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
|
||||
sccache --show-stats; \
|
||||
fi
|
||||
|
||||
# -----------------------
|
||||
# Shared vLLM build dependencies
|
||||
#
|
||||
# Install requirements once, before bringing source-specific inputs into either
|
||||
# consumer. Native compilation and wheel packaging then share this expensive
|
||||
# layer while retaining their independent source cache boundaries.
|
||||
FROM base AS build_vllm_dependencies
|
||||
ARG COMMON_WORKDIR
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY --from=fetch_vllm \
|
||||
${COMMON_WORKDIR}/vllm/requirements/common.txt \
|
||||
${COMMON_WORKDIR}/vllm/requirements/rocm.txt \
|
||||
requirements/
|
||||
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
# -----------------------
|
||||
# vLLM native build stages
|
||||
#
|
||||
# csrc-build intentionally copies only files that affect ROCm native extension
|
||||
# compilation. That keeps unrelated CI/test/docs edits from invalidating the
|
||||
# expensive HIP/C++ build layer.
|
||||
FROM base AS csrc-build
|
||||
ARG COMMON_WORKDIR
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY requirements/rocm.txt requirements/rocm.txt
|
||||
COPY requirements/common.txt requirements/common.txt
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
FROM build_vllm_dependencies AS csrc-build
|
||||
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
|
||||
# but it still participates in the BuildKit cache key for this RUN step.
|
||||
COPY setup.py CMakeLists.txt ./
|
||||
@@ -277,7 +288,6 @@ COPY csrc csrc/
|
||||
COPY vllm/envs.py vllm/envs.py
|
||||
COPY vllm/__init__.py vllm/__init__.py
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
|
||||
|
||||
RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
|
||||
@@ -296,11 +306,10 @@ RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproj
|
||||
|
||||
# Build the full vLLM ROCm wheel by reusing the native extension wheel from
|
||||
# csrc-build. This stage still rebuilds for Python/package changes, but skips
|
||||
# the expensive HIP/C++ compile when native inputs are unchanged.
|
||||
FROM fetch_vllm AS build_vllm
|
||||
ARG COMMON_WORKDIR
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
|
||||
# the expensive dependency installation and HIP/C++ compilation when their
|
||||
# respective inputs are unchanged.
|
||||
FROM build_vllm_dependencies AS build_vllm
|
||||
COPY --from=fetch_vllm ${COMMON_WORKDIR}/vllm ${COMMON_WORKDIR}/vllm
|
||||
COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
|
||||
|
||||
# Drop the pre-built Rust artifacts into the source tree. setup.py detects
|
||||
@@ -308,9 +317,7 @@ COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
|
||||
COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
|
||||
COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/
|
||||
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
cd vllm \
|
||||
&& uv pip install --system -r requirements/rocm.txt \
|
||||
RUN cd "${COMMON_WORKDIR}/vllm" \
|
||||
&& export VLLM_USE_PRECOMPILED=1 \
|
||||
&& export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
|
||||
&& export VLLM_DOCKER_BUILD_CONTEXT=1 \
|
||||
|
||||
+47
-16
@@ -134,11 +134,7 @@ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
|
||||
###
|
||||
FROM base AS build_pytorch
|
||||
ARG PYTORCH_BRANCH
|
||||
ARG PYTORCH_VISION_BRANCH
|
||||
ARG PYTORCH_AUDIO_BRANCH
|
||||
ARG PYTORCH_REPO
|
||||
ARG PYTORCH_VISION_REPO
|
||||
ARG PYTORCH_AUDIO_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN apt-get update && apt-get install -y pkg-config liblzma-dev
|
||||
@@ -156,8 +152,27 @@ RUN cd pytorch && python3 tools/amd_build/build_amd.py \
|
||||
&& sccache --show-stats; \
|
||||
fi \
|
||||
&& CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
### Shared PyTorch runtime for dependent builds
|
||||
###
|
||||
FROM base AS build_pytorch_runtime
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \
|
||||
pip install /install/*.whl
|
||||
|
||||
|
||||
###
|
||||
### TorchVision build
|
||||
###
|
||||
FROM build_pytorch_runtime AS build_torchvision
|
||||
ARG PYTORCH_VISION_BRANCH
|
||||
ARG PYTORCH_VISION_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN git clone ${PYTORCH_VISION_REPO} vision
|
||||
RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
@@ -166,23 +181,31 @@ RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
|
||||
&& export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
|
||||
fi \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
### TorchAudio build
|
||||
###
|
||||
FROM build_pytorch_runtime AS build_torchaudio
|
||||
ARG PYTORCH_AUDIO_BRANCH
|
||||
ARG PYTORCH_AUDIO_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN git clone ${PYTORCH_AUDIO_REPO} audio
|
||||
RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
|
||||
&& git submodule update --init --recursive \
|
||||
&& pip install -r requirements.txt \
|
||||
&& git submodule update --init --recursive
|
||||
RUN cd audio && pip install -r requirements.txt
|
||||
RUN cd audio \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
export HIP_CLANG_PATH=/opt/sccache-wrappers \
|
||||
&& export CMAKE_C_COMPILER_LAUNCHER=sccache \
|
||||
&& export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
|
||||
fi \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
|
||||
&& cp /app/vision/dist/*.whl /app/install \
|
||||
&& cp /app/audio/dist/*.whl /app/install
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
@@ -263,6 +286,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
|
||||
@@ -277,6 +304,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
|
||||
@@ -315,4 +346,4 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
|
||||
&& echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
|
||||
&& echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
|
||||
&& echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
|
||||
|
||||
+15
-24
@@ -338,33 +338,22 @@ group "test-rocm-ci-with-artifacts" {
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed
|
||||
# per-commit image is useful.
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed,
|
||||
# build-scoped image is useful.
|
||||
group "test-rocm-ci-with-wheel" {
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
|
||||
# to the primary tag for this build. Builds always publish a content-scoped tag
|
||||
# when the ci_base content hash is available. Builds with BUILDKITE_COMMIT also
|
||||
# publish a commit-scoped tag, either as the primary tag or an additional alias.
|
||||
# NIGHTLY=1 builds on the stable branch can additionally set
|
||||
# CI_BASE_IMAGE_TAG_STABLE to refresh rocm/vllm-dev:ci_base.
|
||||
# Primary output tag for the ci_base build. In Buildkite this is a unique,
|
||||
# build-scoped tag; ci-bake-rocm.sh validates it before creating content and
|
||||
# stable aliases.
|
||||
variable "CI_BASE_IMAGE_TAG" {
|
||||
default = "rocm/vllm-dev:ci_base"
|
||||
}
|
||||
|
||||
# Supplemental tags only. ci-bake-rocm.sh leaves these empty when the same ref
|
||||
# is already the primary CI_BASE_IMAGE_TAG.
|
||||
variable "CI_BASE_IMAGE_TAG_COMMIT_EXTRA" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "CI_BASE_IMAGE_TAG_CONTENT_EXTRA" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "CI_BASE_IMAGE_TAG_STABLE" {
|
||||
# Versioned, content-addressed trusted ref. Preview builds import it read-only;
|
||||
# the wrapper creates their source-scoped content alias after validation.
|
||||
variable "CI_BASE_TRUSTED_CONTENT_REF" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
@@ -403,21 +392,23 @@ target "deepep-rocm-ci" {
|
||||
# Uses inline cache metadata on the ci_base image itself instead of exporting a
|
||||
# separate registry cache artifact.
|
||||
target "ci-base-rocm-ci" {
|
||||
inherits = ["_common-rocm", "_ci-rocm", "_labels"]
|
||||
inherits = ["_common-rocm"]
|
||||
target = "ci_base"
|
||||
args = {
|
||||
max_jobs = CI_MAX_JOBS
|
||||
}
|
||||
cache-from = concat(
|
||||
compact([
|
||||
CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "",
|
||||
CI_BASE_IMAGE_TAG_COMMIT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_COMMIT_EXTRA}" : "",
|
||||
CI_BASE_IMAGE_TAG_CONTENT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_CONTENT_EXTRA}" : "",
|
||||
CI_BASE_IMAGE_TAG_STABLE != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_STABLE}" : "",
|
||||
CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "",
|
||||
]),
|
||||
# Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages
|
||||
# are cache hits even when ci_base itself needs rebuilding.
|
||||
get_cache_from_rocm_deps(),
|
||||
)
|
||||
cache-to = ["type=inline"]
|
||||
tags = compact([CI_BASE_IMAGE_TAG, CI_BASE_IMAGE_TAG_COMMIT_EXTRA, CI_BASE_IMAGE_TAG_CONTENT_EXTRA, CI_BASE_IMAGE_TAG_STABLE])
|
||||
tags = compact([CI_BASE_IMAGE_TAG])
|
||||
attest = ["type=provenance,disabled=true"]
|
||||
output = ["type=registry"]
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user