From b12e42d13277a691226e0f6654ad04f790acadd6 Mon Sep 17 00:00:00 2001 From: xiangdong <40376367+zxd1997066@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:20:32 +0800 Subject: [PATCH] [XPU][CI] Refine docker image build and pull/create lock mechanism in Intel GPU CI (#44481) Signed-off-by: zengxian --- .../scripts/hardware_ci/run-intel-test.sh | 24 +-- docker/Dockerfile.xpu | 171 ++++++++++-------- 2 files changed, 97 insertions(+), 98 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh index 0605f23e75..0eb93f5a6b 100755 --- a/.buildkite/scripts/hardware_ci/run-intel-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh @@ -324,23 +324,6 @@ IMAGE="${IMAGE_TAG_XPU:-${image_name}}" echo "Using image: ${IMAGE}" -if docker image inspect "${IMAGE}" >/dev/null 2>&1; then - echo "Image already exists locally, skipping pull" -else - echo "Image not found locally, waiting for lock..." - - flock /tmp/docker-pull.lock bash -c " - if docker image inspect '${IMAGE}' >/dev/null 2>&1; then - echo 'Image already pulled by another runner' - else - echo 'Pulling image...' - timeout 900 docker pull '${IMAGE}' - fi - " - - echo "Pull step completed" -fi - remove_docker_container() { docker rm -f "${container_name}" || true } @@ -357,9 +340,12 @@ export HF_TOKEN ZE_AFFINITY_MASK { flock 9 - if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then - echo 'Image missing before container creation, pulling again...' + if docker image inspect "${IMAGE}" >/dev/null 2>&1; then + echo "Image already exists locally, skipping pull" + else + echo "Image not found locally, pulling image..." timeout 900 docker pull "${IMAGE}" + echo "Pull step completed" fi docker create \ diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index ef05b4aa2e..9f129734ce 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -89,8 +89,8 @@ RUN mkdir neo && \ ENV PATH="/root/.local/bin:$PATH" ENV VIRTUAL_ENV="/opt/venv" ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python -RUN curl -LsSf https://astral.sh/uv/install.sh | sh -RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} +RUN curl -LsSf https://astral.sh/uv/install.sh | sh \ + && uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} ENV PATH="$VIRTUAL_ENV/bin:$PATH" # This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3. @@ -99,8 +99,8 @@ RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \ rm "${ONECCL_INSTALLER}" && \ echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \ - echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc -RUN rm -f /opt/intel/oneapi/ccl/latest && \ + echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc && \ + rm -f /opt/intel/oneapi/ccl/latest && \ ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest SHELL ["bash", "-c"] @@ -119,19 +119,98 @@ ENV UV_LINK_MODE="copy" RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,src=requirements/common.txt,target=/workspace/vllm/requirements/common.txt \ --mount=type=bind,src=requirements/xpu.txt,target=/workspace/vllm/requirements/xpu.txt \ - --mount=type=bind,src=requirements/test/xpu.txt,target=/workspace/vllm/requirements/test/xpu.txt \ - uv pip install --upgrade pip && \ - uv pip install -r requirements/xpu.txt && \ - uv pip install grpcio-tools protobuf nanobind && \ - source /opt/intel/oneapi/setvars.sh --force && \ - source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \ - export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \ - uv pip install --no-build-isolation -r /workspace/vllm/requirements/test/xpu.txt + uv pip install --upgrade pip ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/" +CMD ["/bin/bash"] +######################### UCX + NIXL BUILD STAGE ######################### +# Build UCX and NIXL in a dedicated stage so compiler/autotools layers are +# never included in the final runtime image (mirrors ROCm's build_rixl stage). +FROM vllm-base AS ucx-nixl-build + +ARG UCX_VERSION=e5d98879705239d254ede40b4a52891850cb5349 +ARG NIXL_VERSION=0.7.0 + +# Build-time only: compiler, autotools, and verbs dev headers +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + build-essential \ + autoconf \ + automake \ + libtool \ + pkg-config \ + libibverbs-dev \ + librdmacm-dev \ + && rm -rf /var/lib/apt/lists/* + +# Build UCX and produce a NIXL wheel so the final image needs no compiler. +# patchelf (installed via uv) is used by the NIXL wheel build to rewrite +# RPATH entries, making the wheel portable across stages. +RUN --mount=type=cache,target=/root/.cache/uv \ + git clone https://github.com/openucx/ucx /tmp/ucx_source && \ + cd /tmp/ucx_source && git checkout "${UCX_VERSION}" && \ + bash autogen.sh && \ + ./configure --prefix=/tmp/ucx_install --with-ze=yes --enable-examples --enable-mt && \ + make CFLAGS="-Wno-error=incompatible-pointer-types" -j8 && make install && \ + git clone https://github.com/ai-dynamo/nixl /tmp/nixl_source && \ + cd /tmp/nixl_source && git checkout "${NIXL_VERSION}" && \ + uv pip install --upgrade meson pybind11 patchelf && \ + uv pip install -r requirements.txt && \ + PKG_CONFIG_PATH=/tmp/ucx_install/lib/pkgconfig \ + LD_LIBRARY_PATH=/tmp/ucx_install/lib \ + python -m pip wheel --no-deps . -w /tmp/nixl_wheels/ && \ + find /tmp/ucx_install -type f \( -name '*.a' -o -name '*.la' \) -delete && \ + rm -rf /tmp/ucx_install/include /tmp/ucx_install/share /tmp/ucx_install/etc /tmp/ucx_install/lib/cmake /tmp/ucx_install/bin && \ + rm -rf /tmp/ucx_source /tmp/nixl_source + +FROM vllm-base AS vllm-openai + +ARG UCX_VERSION=e5d98879705239d254ede40b4a52891850cb5349 +ARG NIXL_VERSION=0.7.0 + +# Copy compiled UCX runtime libraries and the pre-built NIXL wheel. +# No compiler or autotools are installed in this stage. +COPY --from=ucx-nixl-build /tmp/ucx_install /tmp/ucx_install +COPY --from=ucx-nixl-build /tmp/nixl_wheels /tmp/nixl_wheels + +ENV LD_LIBRARY_PATH=/tmp/ucx_install/lib:${LD_LIBRARY_PATH} + +# Install RDMA runtime libraries (no build tools) and the pre-built NIXL wheel. +# Do not uninstall/reinstall large Python packages here to avoid extra layer +# churn; final package resolution remains in the later app install step. +RUN --mount=type=cache,target=/root/.cache/uv \ + apt-get update -y && apt-get install -y --no-install-recommends \ + rdma-core \ + libibverbs1 \ + librdmacm1 \ + libibumad3 \ + libibmad5 \ + libmlx5-1 \ + libmlx4-1 \ + ibverbs-providers \ + librdmacm1t64 \ + && rm -rf /var/lib/apt/lists/* \ + && uv pip install --no-deps /tmp/nixl_wheels/nixl-*.whl \ + && rm -rf /tmp/nixl_wheels + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,src=requirements/common.txt,target=/workspace/vllm/requirements/common.txt \ + --mount=type=bind,src=requirements/xpu.txt,target=/workspace/vllm/requirements/xpu.txt \ + --mount=type=bind,src=requirements/test/xpu.txt,target=/workspace/vllm/requirements/test/xpu.txt \ + uv pip install grpcio-tools protobuf nanobind && \ + uv pip install -r /workspace/vllm/requirements/xpu.txt && \ + uv pip install --no-build-isolation -r /workspace/vllm/requirements/test/xpu.txt && \ + uv pip uninstall triton triton-xpu && \ + uv pip install triton-xpu==3.7.0 && \ + uv pip uninstall oneccl oneccl-devel && \ + source /opt/intel/oneapi/setvars.sh --force && \ + source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \ + export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" + +# Keep source-dependent layers near the end so frequent code-only changes +# don't invalidate heavy dependency and UCX/NIXL layers. COPY . . # Drop the pre-built rust frontend binary into the source tree. setup.py @@ -147,75 +226,9 @@ ENV VLLM_WORKER_MULTIPROC_METHOD=spawn RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=.git,target=.git \ - uv pip install --no-build-isolation . + uv pip install --no-build-isolation --no-deps . -CMD ["/bin/bash"] - -FROM vllm-base AS vllm-openai - -# install development dependencies (for testing) RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install -e tests/vllm_test_utils -# install NIXL and UCX from source code -ARG UCX_VERSION=e5d98879705239d254ede40b4a52891850cb5349 -ARG NIXL_VERSION=0.7.0 - -RUN apt-get update && apt-get install -y \ - pciutils \ - net-tools \ - iproute2 \ - hwloc \ - numactl \ - wget \ - curl \ - git \ - build-essential \ - autoconf \ - automake \ - libtool \ - pkg-config \ - rdma-core \ - libibverbs-dev \ - ibverbs-utils \ - libibverbs1 \ - librdmacm-dev \ - librdmacm1 \ - libibumad-dev \ - libibumad3 \ - libibmad-dev \ - libibmad5 \ - infiniband-diags \ - perftest \ - ibutils \ - libmlx5-1 \ - libmlx4-1 \ - ibverbs-providers \ - librdmacm1t64 - -ENV PKG_CONFIG_PATH=/tmp/ucx_install/lib/pkgconfig:${PKG_CONFIG_PATH} -ENV LD_LIBRARY_PATH=/tmp/ucx_install/lib:${LD_LIBRARY_PATH} -RUN --mount=type=cache,target=/root/.cache/uv \ - git clone https://github.com/openucx/ucx /tmp/ucx_source && \ - cd /tmp/ucx_source && git checkout "${UCX_VERSION}" && \ - bash autogen.sh && \ - ./configure --prefix=/tmp/ucx_install --with-ze=yes --enable-examples --enable-mt && \ - make CFLAGS="-Wno-error=incompatible-pointer-types" -j8 && make install && \ - git clone https://github.com/ai-dynamo/nixl /tmp/nixl_source && \ - cd /tmp/nixl_source && git checkout "${NIXL_VERSION}" && \ - cd /tmp/nixl_source && \ - uv pip install --upgrade meson pybind11 patchelf && \ - uv pip install -r requirements.txt && \ - uv pip install . && \ - rm -rf /tmp/ucx_source /tmp/nixl_source - -# FIX triton -RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip uninstall triton triton-xpu && \ - uv pip install triton-xpu==3.7.0 - -# remove torch bundled oneccl to avoid conflicts -RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip uninstall oneccl oneccl-devel - ENTRYPOINT ["vllm", "serve"]