[CI] Add registry layer cache to x86 CPU image build (#51911)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-08-12 17:58:01 +08:00
committed by GitHub
parent 8c011da6d0
commit 6accb779a3
2 changed files with 172 additions and 24 deletions
+100 -19
View File
@@ -19,8 +19,42 @@
# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
#
######################### BASE IMAGE #########################
# Common apt packages and the optional sccache binary install, shared by
# base-common and rust-build (the latter is deliberately not `FROM
# base-common`, to stay minimal and build in parallel with vllm-build).
#
# Optional remote (S3-backed) compiler cache. Local BuildKit `--mount=type=cache`
# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
# type=registry`, so they don't survive a build landing on a different/fresh
# builder. sccache's cache lives in S3 instead, so it does.
FROM ubuntu:22.04 AS base
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update -y \
&& apt-get install -y --no-install-recommends ca-certificates curl git
ARG TARGETARCH
ARG USE_SCCACHE
ARG SCCACHE_DOWNLOAD_URL
RUN if [ "$USE_SCCACHE" = "1" ]; then \
echo "Installing sccache..." \
&& case "${TARGETARCH}" in \
arm64) SCCACHE_ARCH="aarch64" ;; \
amd64) SCCACHE_ARCH="x86_64" ;; \
*) echo "Unsupported TARGETARCH for sccache: ${TARGETARCH}" >&2; exit 1 ;; \
esac \
&& export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
&& curl -L -o sccache.tar.gz "${SCCACHE_DOWNLOAD_URL}" \
&& tar -xzf sccache.tar.gz \
&& mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
&& rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
fi
######################### COMMON BASE IMAGE #########################
FROM ubuntu:22.04 AS base-common
FROM base AS base-common
WORKDIR /workspace
@@ -33,11 +67,22 @@ ENV MAX_JOBS=${max_jobs}
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update -y \
&& apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates zlib1g-dev \
&& apt-get install -y --no-install-recommends sudo ccache wget zlib1g-dev \
gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof make xz-utils \
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
&& curl -LsSf https://astral.sh/uv/install.sh | sh
ARG TARGETARCH
ARG USE_SCCACHE
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
ARG SCCACHE_S3_NO_CREDENTIALS=0
ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
# Compiler and linker environment
ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
ENV CCACHE_DIR=/root/.cache/ccache
@@ -63,7 +108,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --upgrade pip && \
uv pip install -r requirements/cpu.txt --torch-backend cpu
ARG TARGETARCH
ENV TARGETARCH=${TARGETARCH}
######################### x86_64 BASE IMAGE #########################
@@ -76,8 +120,8 @@ FROM base-common AS base-arm64
ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
######################### BASE IMAGE #########################
FROM base-${TARGETARCH} AS base
######################### ARCH BASE IMAGE #########################
FROM base-${TARGETARCH} AS base-arch
RUN echo 'ulimit -c 0' >> ~/.bashrc
@@ -85,12 +129,12 @@ RUN echo 'ulimit -c 0' >> ~/.bashrc
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
# with the main vllm-build stage.
FROM ubuntu:22.04 AS rust-build
FROM base AS rust-build
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y \
&& apt-get install -y --no-install-recommends \
ca-certificates curl git build-essential unzip python3 python3-pip \
build-essential unzip python3 python3-pip \
&& rm -rf /var/lib/apt/lists/*
COPY tools/install_protoc.sh /tmp/install_protoc.sh
@@ -101,6 +145,16 @@ WORKDIR /workspace
COPY requirements/build/rust.txt requirements/build/rust.txt
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
# Avoid port collision with vllm-build's own sccache daemon.
ENV SCCACHE_SERVER_PORT=4227
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
# by the wheel build stage.
COPY rust rust
@@ -117,10 +171,20 @@ ENV CARGO_BUILD_JOBS=4
# cache reuse.
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
bash build_rust.sh
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
if [ "$USE_SCCACHE" = "1" ]; then \
export RUSTC_WRAPPER=sccache; \
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
sccache --show-stats; \
fi && \
bash build_rust.sh && \
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
######################### BUILD IMAGE #########################
FROM base AS vllm-build
FROM base-arch AS vllm-build
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
ARG GIT_REPO_CHECK=0
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
@@ -160,17 +224,27 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=cache,target=/root/.cache/ccache \
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
if [ "$USE_SCCACHE" = "1" ]; then \
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
sccache --show-stats; \
fi && \
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 && \
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
######################### TRITON-CPU BUILD IMAGE #########################
FROM base AS vllm-triton-cpu-build
FROM base-arch AS vllm-triton-cpu-build
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so it
# does not inherit the ARG/ENV defined there. Without it, the guard below would
# see an empty value and build triton-cpu on non-x86 targets (e.g. arm64).
# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
# so it does not inherit the ARG/ENV defined there. Without it, the guard
# below would see an empty value and build triton-cpu on non-x86 targets
# (e.g. arm64).
ARG VLLM_CPU_X86=0
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
WORKDIR /vllm-workspace
RUN mkdir dist
@@ -178,15 +252,22 @@ RUN mkdir dist
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=cache,target=/root/.cache/ccache \
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
git clone --recurse-submodules "https://github.com/triton-lang/triton-cpu.git"; \
cd triton-cpu; \
git checkout "270e696d"; \
if [ "$USE_SCCACHE" = "1" ]; then \
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
sccache --show-stats; \
fi; \
uv build --wheel --out-dir=../dist; \
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
fi
######################### TEST DEPS #########################
FROM base AS vllm-test-deps
FROM base-arch AS vllm-test-deps
WORKDIR /vllm-workspace
@@ -260,11 +341,11 @@ ENV HF_XET_HIGH_PERFORMANCE 1
ENV HF_HUB_DOWNLOAD_TIMEOUT 60
######################### RELEASE IMAGE #########################
FROM base AS vllm-openai
FROM base-arch AS vllm-openai
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so the
# RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 would
# otherwise see an empty value and try to install it on non-x86 targets.
# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
# so the RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86
# would otherwise see an empty value and try to install it on non-x86 targets.
ARG VLLM_CPU_X86=0
WORKDIR /vllm-workspace