[ROCm][CI] Optimize ROCm Docker build: registry cache, DeepEP, and ci-bake script (#36949)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
+200
-108
@@ -2,6 +2,7 @@
|
||||
ARG REMOTE_VLLM="0"
|
||||
ARG COMMON_WORKDIR=/app
|
||||
ARG BASE_IMAGE=rocm/vllm-dev:base
|
||||
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
|
||||
# NIC backend for MoRI RDMA support.
|
||||
# By default (all), drivers and userspace libraries for all supported NIC types
|
||||
# (ainic and bnxt) are installed; MoRI selects the appropriate one at runtime.
|
||||
@@ -16,7 +17,8 @@ ARG NIC_BACKEND=all
|
||||
ARG AINIC_VERSION=1.117.3-hydra
|
||||
ARG UBUNTU_CODENAME=jammy
|
||||
|
||||
# Sccache configuration (only used in release pipeline)
|
||||
# Sccache configuration. Release builds use this today; CI can opt in when a
|
||||
# shared S3-compatible cache backend is available.
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_DOWNLOAD_URL
|
||||
ARG SCCACHE_ENDPOINT
|
||||
@@ -29,12 +31,16 @@ FROM ${BASE_IMAGE} AS base
|
||||
ARG ARG_PYTORCH_ROCM_ARCH
|
||||
ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
|
||||
|
||||
# Install some basic utilities
|
||||
# Install build dependencies and utilities
|
||||
RUN apt-get update -q -y && apt-get install -q -y \
|
||||
sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \
|
||||
apt-transport-https ca-certificates wget curl \
|
||||
libnuma-dev
|
||||
RUN python3 -m pip install --upgrade pip
|
||||
libnuma-dev ccache mold
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
python3 -m pip install --upgrade pip
|
||||
# Note: mold is installed but not set as the system default linker because
|
||||
# some packages use JIT compilation at runtime with flags mold does not support.
|
||||
# Build stages opt in via LDFLAGS="-fuse-ld=mold".
|
||||
# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_base)
|
||||
ARG USE_SCCACHE
|
||||
RUN if [ "$USE_SCCACHE" != "1" ]; then \
|
||||
@@ -55,6 +61,12 @@ ENV UV_HTTP_TIMEOUT=500
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
# Use copy mode to avoid hardlink failures with Docker cache mounts
|
||||
ENV UV_LINK_MODE=copy
|
||||
# ccache directory - persisted across layer rebuilds via cache mounts.
|
||||
ENV CCACHE_DIR=/root/.cache/ccache
|
||||
ENV CCACHE_COMPILERCHECK=content
|
||||
# Empty by default so build steps fall back to $(nproc); CI can override.
|
||||
ARG max_jobs
|
||||
ENV MAX_JOBS=${max_jobs}
|
||||
|
||||
# Install sccache if USE_SCCACHE is enabled (for release builds)
|
||||
ARG USE_SCCACHE
|
||||
@@ -86,6 +98,7 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
ARG USE_SCCACHE
|
||||
ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
|
||||
ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
|
||||
ENV SCCACHE_ENDPOINT=${USE_SCCACHE:+${SCCACHE_ENDPOINT}}
|
||||
ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
|
||||
ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
|
||||
|
||||
@@ -114,8 +127,7 @@ FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
|
||||
# -----------------------
|
||||
# Rust build stage
|
||||
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
||||
# don't need the rust toolchain or protoc. Runs in parallel with the main wheel
|
||||
# build for faster end-to-end builds.
|
||||
# don't need the rust toolchain or protoc.
|
||||
FROM fetch_vllm AS rust-build
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
@@ -144,24 +156,74 @@ ENV RUSTUP_MAX_RETRIES=10
|
||||
# layer for later COPY --from=rust-build.
|
||||
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
|
||||
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
|
||||
--mount=type=cache,id=vllm-rocm-cargo-target,target=${COMMON_WORKDIR}/vllm/rust/target,sharing=locked \
|
||||
cd ${COMMON_WORKDIR}/vllm \
|
||||
&& VLLM_RS_TARGET_PATH=/tmp/vllm-rs bash build_rust.sh \
|
||||
&& test -x /tmp/vllm-rs
|
||||
|
||||
# -----------------------
|
||||
# vLLM build stages
|
||||
# vLLM native build stages
|
||||
#
|
||||
# csrc-build intentionally copies only files that affect ROCm native extension
|
||||
# compilation. That keeps unrelated CI/test/docs edits from invalidating the
|
||||
# expensive HIP/C++ build layer.
|
||||
FROM base AS csrc-build
|
||||
ARG COMMON_WORKDIR
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY requirements/rocm.txt requirements/rocm.txt
|
||||
COPY requirements/common.txt requirements/common.txt
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do
|
||||
# not invalidate the expensive native build layer.
|
||||
COPY setup.py CMakeLists.txt ./
|
||||
COPY cmake cmake/
|
||||
COPY csrc csrc/
|
||||
COPY vllm/envs.py vllm/envs.py
|
||||
COPY vllm/__init__.py vllm/__init__.py
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
|
||||
|
||||
RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
|
||||
--mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
|
||||
export CCACHE_BASEDIR="$PWD" \
|
||||
&& echo "=== ccache stats before ROCm native build ===" \
|
||||
&& (ccache --show-stats || true) \
|
||||
&& (ccache --zero-stats || true) \
|
||||
&& EFFECTIVE_MAX_JOBS="${MAX_JOBS:-$(nproc)}" \
|
||||
&& echo "Building ROCm native extension wheel with MAX_JOBS=${EFFECTIVE_MAX_JOBS}" \
|
||||
&& LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& test -d dist \
|
||||
&& ls dist/*.whl >/dev/null \
|
||||
&& echo "=== ccache stats after ROCm native build ===" \
|
||||
&& (ccache --show-stats || true)
|
||||
|
||||
# Build the full vLLM ROCm wheel by reusing the native extension wheel from
|
||||
# csrc-build. This stage still rebuilds for Python/package changes, but skips
|
||||
# the expensive HIP/C++ compile when native inputs are unchanged.
|
||||
FROM fetch_vllm AS build_vllm
|
||||
ARG COMMON_WORKDIR
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
|
||||
COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /tmp/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
|
||||
|
||||
# Build vLLM (setup.py auto-detects sccache in PATH)
|
||||
RUN cd vllm \
|
||||
&& python3 -m pip install -r requirements/rocm.txt \
|
||||
&& python3 setup.py clean --all \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
cd vllm \
|
||||
&& uv pip install --system -r requirements/rocm.txt \
|
||||
&& export VLLM_USE_PRECOMPILED=1 \
|
||||
&& export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
|
||||
&& export VLLM_DOCKER_BUILD_CONTEXT=1 \
|
||||
&& echo "Packaging vLLM ROCm wheel using precompiled extensions from ${VLLM_PRECOMPILED_WHEEL_LOCATION}" \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& test -d dist \
|
||||
&& ls dist/*.whl >/dev/null
|
||||
FROM scratch AS export_vllm
|
||||
ARG COMMON_WORKDIR
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/dist/*.whl /
|
||||
@@ -171,6 +233,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
|
||||
|
||||
# RIXL/UCX build stages
|
||||
@@ -201,14 +264,17 @@ RUN apt-get -y update && apt-get -y install autoconf libtool pkg-config \
|
||||
ibverbs-providers \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN uv pip install --system meson auditwheel patchelf tomlkit
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system meson auditwheel patchelf tomlkit
|
||||
|
||||
RUN cd /usr/local/src && \
|
||||
RUN --mount=type=cache,target=/root/.cache/ccache \
|
||||
cd /usr/local/src && \
|
||||
git clone ${UCX_REPO} && \
|
||||
cd ucx && \
|
||||
git checkout ${UCX_BRANCH} && \
|
||||
./autogen.sh && \
|
||||
mkdir build && cd build && \
|
||||
CC="ccache gcc" CXX="ccache g++" \
|
||||
../configure \
|
||||
--prefix=/usr/local/ucx \
|
||||
--enable-shared \
|
||||
@@ -220,20 +286,22 @@ RUN cd /usr/local/src && \
|
||||
--with-verbs \
|
||||
--with-dm \
|
||||
--enable-mt && \
|
||||
make -j && \
|
||||
make -j$(nproc) && \
|
||||
make install
|
||||
|
||||
ENV PATH=/usr/local/ucx/bin:$PATH
|
||||
ENV LD_LIBRARY_PATH=${UCX_HOME}/lib:${LD_LIBRARY_PATH}
|
||||
|
||||
RUN git clone ${RIXL_REPO} /opt/rixl && \
|
||||
RUN --mount=type=cache,target=/root/.cache/ccache \
|
||||
git clone ${RIXL_REPO} /opt/rixl && \
|
||||
cd /opt/rixl && \
|
||||
git checkout ${RIXL_BRANCH} && \
|
||||
CC="ccache gcc" CXX="ccache g++" \
|
||||
meson setup build --prefix=${RIXL_HOME} \
|
||||
-Ducx_path=${UCX_HOME} \
|
||||
-Drocm_path=${ROCM_PATH} && \
|
||||
cd build && \
|
||||
ninja && \
|
||||
ninja -j$(nproc) && \
|
||||
ninja install
|
||||
|
||||
# Generate RIXL wheel
|
||||
@@ -250,30 +318,44 @@ RUN cd /opt/rixl && \
|
||||
--ucx-plugins-dir ${UCX_HOME}/lib/ucx \
|
||||
--nixl-plugins-dir ${RIXL_HOME}/lib/x86_64-linux-gnu/plugins
|
||||
|
||||
# DeepEP build stage
|
||||
FROM base AS build_deep
|
||||
# ROCShmem build stage - split from DeepEP so changing DEEPEP_BRANCH does not
|
||||
# invalidate the slow ROCShmem build.
|
||||
FROM base AS build_rocshmem
|
||||
ARG ROCSHMEM_BRANCH="f0acb0c6"
|
||||
ARG ROCSHMEM_REPO="https://github.com/ROCm/rocm-systems.git"
|
||||
ARG DEEPEP_BRANCH="a9ea9774"
|
||||
ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git"
|
||||
ARG DEEPEP_NIC="cx7"
|
||||
# DeepEP only supports gfx942 and gfx950; build ROCShmem for the same set so
|
||||
# it can be linked against DeepEP without arch mismatches.
|
||||
ARG DEEPEP_ROCM_ARCH="gfx942;gfx950"
|
||||
ENV ROCM_PATH=/opt/rocm
|
||||
ENV ROCSHMEM_DIR=/opt/rocshmem
|
||||
|
||||
RUN git clone ${ROCSHMEM_REPO} \
|
||||
RUN --mount=type=cache,target=/root/.cache/ccache \
|
||||
git clone --no-checkout --filter=blob:none ${ROCSHMEM_REPO} \
|
||||
&& cd rocm-systems \
|
||||
&& git sparse-checkout set --cone projects/rocshmem \
|
||||
&& git checkout ${ROCSHMEM_BRANCH} \
|
||||
&& mkdir -p projects/rocshmem/build \
|
||||
&& cd projects/rocshmem/build \
|
||||
&& INSTALL_PREFIX=${ROCSHMEM_DIR} \
|
||||
../scripts/build_configs/all_backends -DUSE_EXTERNAL_MPI=OFF
|
||||
&& CC="ccache gcc" CXX="ccache g++" INSTALL_PREFIX=${ROCSHMEM_DIR} \
|
||||
bash ../scripts/build_configs/all_backends \
|
||||
-DROCM_PATH=${ROCM_PATH} \
|
||||
-DGPU_TARGETS="${DEEPEP_ROCM_ARCH}" \
|
||||
-DUSE_EXTERNAL_MPI=OFF
|
||||
|
||||
# Build DeepEP wheel.
|
||||
# DeepEP looks for rocshmem at ROCSHMEM_DIR.
|
||||
RUN git clone ${DEEPEP_REPO} \
|
||||
# DeepEP build stage - depends on ROCShmem, builds the HIP kernel wheel.
|
||||
FROM build_rocshmem AS build_deepep
|
||||
ARG DEEPEP_BRANCH="a9ea9774"
|
||||
ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git"
|
||||
ARG DEEPEP_NIC="cx7"
|
||||
|
||||
# Build DeepEP wheel. DeepEP looks for rocshmem at ROCSHMEM_DIR.
|
||||
# DeepEP only supports gfx942 and gfx950, so avoid gfx90a in the default list.
|
||||
RUN --mount=type=cache,target=/root/.cache/ccache \
|
||||
export PYTORCH_ROCM_ARCH="gfx942;gfx950" \
|
||||
&& git clone ${DEEPEP_REPO} \
|
||||
&& cd DeepEP \
|
||||
&& git checkout ${DEEPEP_BRANCH} \
|
||||
&& python3 setup.py --variant rocm --rocm-explicit-ctx --nic ${DEEPEP_NIC} bdist_wheel --dist-dir=/app/deep_install
|
||||
&& LDFLAGS="-fuse-ld=mold" MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py --variant rocm --rocm-explicit-ctx --nic ${DEEPEP_NIC} bdist_wheel --dist-dir=/app/deep_install
|
||||
|
||||
# MoRI runtime dependencies live in Dockerfile.rocm so NIC backend changes do
|
||||
# not force users to rebuild the long-lived Dockerfile.rocm_base image.
|
||||
@@ -372,8 +454,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \
|
||||
# Extract version from git BEFORE any modifications (pin_rocm_dependencies.py modifies requirements/rocm.txt)
|
||||
# This ensures setuptools_scm sees clean repo state for version detection
|
||||
RUN --mount=type=bind,source=.git,target=vllm/.git \
|
||||
--mount=type=cache,target=/root/.cache/uv \
|
||||
cd vllm \
|
||||
&& pip install setuptools_scm regex \
|
||||
&& uv pip install --system setuptools_scm regex \
|
||||
&& VLLM_VERSION=$(python3 -c "import setuptools_scm; print(setuptools_scm.get_version())") \
|
||||
&& echo "Detected vLLM version: ${VLLM_VERSION}" \
|
||||
&& echo "${VLLM_VERSION}" > /tmp/vllm_version.txt
|
||||
@@ -409,18 +492,20 @@ RUN echo "Pinning vLLM dependencies to custom wheel versions..." \
|
||||
&& python3 /tmp/pin_rocm_dependencies.py /install ${COMMON_WORKDIR}/vllm/requirements/rocm.txt
|
||||
|
||||
# Install dependencies using custom wheels from /install
|
||||
RUN cd vllm \
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
cd vllm \
|
||||
&& echo "Building vLLM with custom wheels from /install" \
|
||||
&& python3 -m pip install --find-links /install -r requirements/rocm.txt \
|
||||
&& python3 setup.py clean --all
|
||||
&& uv pip install --system --find-links /install -r requirements/rocm.txt
|
||||
|
||||
# Build wheel using pre-extracted version to avoid dirty state from modified requirements/rocm.txt
|
||||
# (setup.py auto-detects sccache in PATH)
|
||||
# (setup.py auto-detects ccache/sccache in PATH)
|
||||
RUN --mount=type=bind,source=.git,target=vllm/.git \
|
||||
--mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
|
||||
cd vllm \
|
||||
&& export CCACHE_BASEDIR="$PWD" \
|
||||
&& export SETUPTOOLS_SCM_PRETEND_VERSION=$(cat /tmp/vllm_version.txt) \
|
||||
&& echo "Building wheel with version: ${SETUPTOOLS_SCM_PRETEND_VERSION}" \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist
|
||||
&& MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py bdist_wheel --dist-dir=dist
|
||||
|
||||
FROM scratch AS export_vllm_wheel_release
|
||||
ARG COMMON_WORKDIR
|
||||
@@ -431,112 +516,118 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
|
||||
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
|
||||
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
|
||||
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
|
||||
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml
|
||||
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
|
||||
|
||||
# -----------------------
|
||||
# Test vLLM image
|
||||
FROM mori_base AS test
|
||||
# CI base image (Tier 1) - stable, rarely changing CI dependencies.
|
||||
# Per-PR test builds pull this as CI_BASE_IMAGE so the test stage only layers
|
||||
# in the vLLM artifacts for the current commit.
|
||||
FROM mori_base AS ci_base
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install vLLM using uv (inherited from base stage)
|
||||
# Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version
|
||||
RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
|
||||
--mount=type=cache,target=/root/.cache/uv \
|
||||
cd /install \
|
||||
&& uv pip install --system -r requirements/rocm.txt \
|
||||
&& uv pip install --system -r requirements/test/rocm.txt \
|
||||
&& pip uninstall -y vllm \
|
||||
&& uv pip install --system *.whl
|
||||
|
||||
# Persist the built wheel in the image so python_only_compile_rocm.sh can
|
||||
# reinstall it after removing compilers. The bind-mounted /install contents
|
||||
# above are not available once that RUN step completes.
|
||||
COPY --from=export_vllm /*.whl /opt/vllm-wheels/
|
||||
|
||||
# Update rdma-core to support latest rocshmem
|
||||
# Update rdma-core to support latest rocshmem.
|
||||
ARG DEEPEP_NIC
|
||||
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
|
||||
git clone --branch v62.0 --depth 1 https://github.com/linux-rdma/rdma-core.git /tmp/rdma-core && \
|
||||
cd /tmp/rdma-core && \
|
||||
mkdir -p build && cd build && \
|
||||
cmake -GNinja -DCMAKE_INSTALL_PREFIX=/usr -DNO_MAN_PAGES=1 .. && \
|
||||
ninja && ninja install && ldconfig && rm -rf /tmp/rdma-core; \
|
||||
ninja && ninja install && ldconfig && rm -rf /tmp/rdma-core; \
|
||||
fi
|
||||
|
||||
# Install RIXL wheel
|
||||
# Install RIXL + DeepEP wheels.
|
||||
RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \
|
||||
uv pip install --system /rixl_install/*.whl
|
||||
--mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \
|
||||
uv pip install --system /rixl_install/*.whl /deep_install/*.whl
|
||||
|
||||
# Install DeepEP wheel
|
||||
RUN --mount=type=bind,from=build_deep,src=/app/deep_install,target=/deep_install \
|
||||
uv pip install --system /deep_install/*.whl
|
||||
COPY --from=build_deep /opt/rocshmem /opt/rocshmem
|
||||
# Copy ROCShmem runtime libraries.
|
||||
COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
|
||||
|
||||
# RIXL/MoRIIO runtime dependencies (RDMA userspace libraries)
|
||||
RUN apt-get update -q -y && apt-get install -q -y \
|
||||
# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
|
||||
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
|
||||
librdmacm1 \
|
||||
libibverbs1 \
|
||||
ibverbs-providers \
|
||||
ibverbs-utils \
|
||||
pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
|
||||
libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
ARG COMMON_WORKDIR
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
|
||||
|
||||
# install development dependencies (for testing)
|
||||
RUN cd /vllm-workspace \
|
||||
&& python3 -m pip install -e tests/vllm_test_utils \
|
||||
&& python3 -m pip install pytest-shard
|
||||
|
||||
# enable fast downloads from hf (for testing)
|
||||
ENV HF_XET_HIGH_PERFORMANCE=1
|
||||
|
||||
# increase timeout for hf downloads (for testing)
|
||||
ENV HF_HUB_DOWNLOAD_TIMEOUT 60
|
||||
|
||||
# install audio decode package `torchcodec` from source (required due to
|
||||
# ROCm and torch version mismatch) for tests with datasets package
|
||||
# Install torchcodec from source for ROCm/torch ABI compatibility.
|
||||
COPY tools/install_torchcodec_rocm.sh /tmp/install_torchcodec.sh
|
||||
RUN bash /tmp/install_torchcodec.sh \
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
--mount=type=cache,target=/root/.cache/torchcodec-wheels \
|
||||
bash /tmp/install_torchcodec.sh \
|
||||
&& rm /tmp/install_torchcodec.sh \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy in the v1 package (for python-only install test group)
|
||||
COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
|
||||
# Pre-install shared ROCm runtime dependencies.
|
||||
COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system -r /tmp/ci-base-requirements/rocm.txt \
|
||||
&& rm -rf /tmp/ci-base-requirements
|
||||
|
||||
# Set MIOPEN ENVS to resolve performance regressions in MIOpen 3D convolution kernel
|
||||
# Enable fast and less brittle model downloads in tests.
|
||||
ENV HF_XET_HIGH_PERFORMANCE=1
|
||||
ENV HF_HUB_DOWNLOAD_TIMEOUT=60
|
||||
|
||||
# Pre-install vLLM test dependencies.
|
||||
COPY requirements/test/rocm.txt /tmp/rocm-test-reqs.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system -r /tmp/rocm-test-reqs.txt
|
||||
|
||||
# Rebuild fastsafetensors from source so its C++ extension is compiled with
|
||||
# USE_ROCM and can detect libamdhip64.so at runtime.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
FASTSAFETENSORS_REQ="$(grep -E '^fastsafetensors(==| @ )' /tmp/rocm-test-reqs.txt | head -1)" \
|
||||
&& test -n "${FASTSAFETENSORS_REQ}" \
|
||||
&& python3 -m pip install --force-reinstall --no-deps \
|
||||
--no-binary fastsafetensors "${FASTSAFETENSORS_REQ}" \
|
||||
&& rm /tmp/rocm-test-reqs.txt
|
||||
|
||||
# Set MIOPEN ENVS to resolve performance regressions in MIOpen 3D convolution kernel.
|
||||
# See: https://github.com/pytorch/pytorch/issues/169857
|
||||
ENV MIOPEN_DEBUG_CONV_DIRECT=0
|
||||
ENV MIOPEN_DEBUG_CONV_GEMM=0
|
||||
|
||||
# Use legacy IPC mode for HSA to avoid GPU memory pinning issues with UCX rocm_ipc
|
||||
# Use legacy IPC mode for HSA to avoid GPU memory pinning issues with UCX rocm_ipc.
|
||||
# See: https://github.com/ROCm/rocm-libraries/issues/6266
|
||||
ENV HSA_ENABLE_IPC_MODE_LEGACY=1
|
||||
|
||||
# Source code is used in the `python_only_compile.sh` test
|
||||
# We hide it inside `src/` so that this source code
|
||||
# will not be imported by other tests
|
||||
RUN mkdir src && mv vllm src/vllm
|
||||
# ROCm profiler limits workaround.
|
||||
RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf
|
||||
ENV KINETO_CONFIG="${COMMON_WORKDIR}/libkineto.conf"
|
||||
|
||||
# This is a workaround to ensure pytest exits with the correct status code in CI tests.
|
||||
RUN printf '%s\n' \
|
||||
'import os' \
|
||||
'' \
|
||||
'_exit_code = 1' \
|
||||
'' \
|
||||
'def pytest_sessionfinish(session, exitstatus):' \
|
||||
' global _exit_code' \
|
||||
' _exit_code = int(exitstatus)' \
|
||||
'' \
|
||||
'def pytest_unconfigure(config):' \
|
||||
' import sys' \
|
||||
' sys.stdout.flush()' \
|
||||
' sys.stderr.flush()' \
|
||||
' os._exit(_exit_code)' \
|
||||
> /vllm-workspace/conftest.py
|
||||
# Install vllm_test_utils in ci_base for ci_base + wheel parity.
|
||||
COPY tests/vllm_test_utils /tmp/vllm_test_utils
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system /tmp/vllm_test_utils \
|
||||
&& rm -rf /tmp/vllm_test_utils
|
||||
|
||||
# -----------------------
|
||||
# Test vLLM image (Tier 2) - vLLM-only layer on top of ci_base.
|
||||
FROM ${CI_BASE_IMAGE} AS test
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
# Install the vLLM wheel (--no-deps: all deps already in ci_base).
|
||||
RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
|
||||
--mount=type=cache,target=/root/.cache/uv \
|
||||
cd /install \
|
||||
&& uv pip install --system --no-deps *.whl
|
||||
|
||||
# Store the vLLM wheel in the image for python-only install tests.
|
||||
COPY --from=export_vllm /*.whl /opt/vllm-wheels/
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
|
||||
|
||||
# Copy in the v1 package (for python-only install test group).
|
||||
COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
|
||||
|
||||
# Hide source under src/ so it won't shadow the installed package in tests.
|
||||
RUN mkdir src && mv vllm src/vllm
|
||||
|
||||
# -----------------------
|
||||
# Final vLLM image
|
||||
@@ -553,6 +644,7 @@ RUN rm -f /usr/bin/sccache || true \
|
||||
# This prevents S3 bucket config from leaking into production images
|
||||
ENV SCCACHE_BUCKET=
|
||||
ENV SCCACHE_REGION=
|
||||
ENV SCCACHE_ENDPOINT=
|
||||
ENV SCCACHE_S3_NO_CREDENTIALS=
|
||||
ENV SCCACHE_IDLE_TIMEOUT=
|
||||
|
||||
|
||||
Reference in New Issue
Block a user