[CI] Add registry layer cache to x86 CPU image build (#51911)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -11,24 +11,91 @@ REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
# replace invalid characters in Docker image tags and truncate to 128 chars
|
||||
clean_docker_tag() {
|
||||
local input="$1"
|
||||
echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
|
||||
}
|
||||
|
||||
# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN
|
||||
# Reuses the same ECR cache repos as the CUDA image build, with an
|
||||
# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated.
|
||||
prepare_cache_tags() {
|
||||
TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
|
||||
MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
|
||||
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
|
||||
if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
|
||||
cache="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
else
|
||||
clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}")
|
||||
cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu"
|
||||
fi
|
||||
CACHE_TO="$cache"
|
||||
CACHE_FROM="$cache"
|
||||
CACHE_FROM_BASE_BRANCH="$cache"
|
||||
else
|
||||
CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
|
||||
CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
|
||||
if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then
|
||||
CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
else
|
||||
clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}")
|
||||
CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu"
|
||||
fi
|
||||
fi
|
||||
|
||||
CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
}
|
||||
|
||||
# authenticate with AWS ECR (public, for the image; private, for the cache repos)
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
|
||||
echo "Image found"
|
||||
else
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build
|
||||
docker build --file docker/Dockerfile.cpu \
|
||||
|
||||
prepare_cache_tags
|
||||
echo "--- :mag: Cache tags"
|
||||
echo "CACHE_TO: ${CACHE_TO}"
|
||||
echo "CACHE_FROM: ${CACHE_FROM}"
|
||||
echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}"
|
||||
echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}"
|
||||
|
||||
# dedupe cache-from refs so identical branch/main fallbacks aren't repeated
|
||||
CACHE_FROM_ARGS=()
|
||||
SEEN_REFS=""
|
||||
for ref in "$CACHE_FROM" "$CACHE_FROM_BASE_BRANCH" "$CACHE_FROM_MAIN"; do
|
||||
if [[ "$SEEN_REFS" != *"|${ref}|"* ]]; then
|
||||
CACHE_FROM_ARGS+=(--cache-from "type=registry,ref=${ref}")
|
||||
SEEN_REFS="${SEEN_REFS}|${ref}|"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- :docker: Setting up buildx"
|
||||
# network=host so the isolated docker-container builder can still reach the
|
||||
# EC2 instance metadata service for sccache's S3 credentials; recreate on
|
||||
# every run in case a stale builder without this driver-opt already exists
|
||||
# on a warm CI host.
|
||||
docker buildx rm vllm-cpu-builder >/dev/null 2>&1 || true
|
||||
docker buildx create --name vllm-cpu-builder --driver docker-container --driver-opt network=host --use
|
||||
docker buildx inspect --bootstrap
|
||||
|
||||
# build and push
|
||||
docker buildx build --file docker/Dockerfile.cpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--build-arg VLLM_CPU_X86=true \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--tag "$IMAGE" \
|
||||
--target vllm-test \
|
||||
"${CACHE_FROM_ARGS[@]}" \
|
||||
--cache-to "type=registry,ref=${CACHE_TO},mode=max" \
|
||||
--push \
|
||||
--progress plain .
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
|
||||
+100
-19
@@ -19,8 +19,42 @@
|
||||
# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
|
||||
#
|
||||
|
||||
######################### BASE IMAGE #########################
|
||||
# Common apt packages and the optional sccache binary install, shared by
|
||||
# base-common and rust-build (the latter is deliberately not `FROM
|
||||
# base-common`, to stay minimal and build in parallel with vllm-build).
|
||||
#
|
||||
# Optional remote (S3-backed) compiler cache. Local BuildKit `--mount=type=cache`
|
||||
# cache mounts (ccache, cargo registry) aren't exported by `--cache-to
|
||||
# type=registry`, so they don't survive a build landing on a different/fresh
|
||||
# builder. sccache's cache lives in S3 instead, so it does.
|
||||
FROM ubuntu:22.04 AS base
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl git
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_DOWNLOAD_URL
|
||||
|
||||
RUN if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
echo "Installing sccache..." \
|
||||
&& case "${TARGETARCH}" in \
|
||||
arm64) SCCACHE_ARCH="aarch64" ;; \
|
||||
amd64) SCCACHE_ARCH="x86_64" ;; \
|
||||
*) echo "Unsupported TARGETARCH for sccache: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac \
|
||||
&& export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
|
||||
&& curl -L -o sccache.tar.gz "${SCCACHE_DOWNLOAD_URL}" \
|
||||
&& tar -xzf sccache.tar.gz \
|
||||
&& mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
|
||||
&& rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \
|
||||
fi
|
||||
|
||||
######################### COMMON BASE IMAGE #########################
|
||||
FROM ubuntu:22.04 AS base-common
|
||||
FROM base AS base-common
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
@@ -33,11 +67,22 @@ ENV MAX_JOBS=${max_jobs}
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates zlib1g-dev \
|
||||
&& apt-get install -y --no-install-recommends sudo ccache wget zlib1g-dev \
|
||||
gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof make xz-utils \
|
||||
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
|
||||
ARG SCCACHE_REGION_NAME=us-west-2
|
||||
ARG SCCACHE_S3_NO_CREDENTIALS=0
|
||||
|
||||
ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
|
||||
ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
|
||||
ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
|
||||
ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
|
||||
|
||||
# Compiler and linker environment
|
||||
ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
|
||||
ENV CCACHE_DIR=/root/.cache/ccache
|
||||
@@ -63,7 +108,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --upgrade pip && \
|
||||
uv pip install -r requirements/cpu.txt --torch-backend cpu
|
||||
|
||||
ARG TARGETARCH
|
||||
ENV TARGETARCH=${TARGETARCH}
|
||||
|
||||
######################### x86_64 BASE IMAGE #########################
|
||||
@@ -76,8 +120,8 @@ FROM base-common AS base-arm64
|
||||
|
||||
ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
|
||||
|
||||
######################### BASE IMAGE #########################
|
||||
FROM base-${TARGETARCH} AS base
|
||||
######################### ARCH BASE IMAGE #########################
|
||||
FROM base-${TARGETARCH} AS base-arch
|
||||
|
||||
RUN echo 'ulimit -c 0' >> ~/.bashrc
|
||||
|
||||
@@ -85,12 +129,12 @@ RUN echo 'ulimit -c 0' >> ~/.bashrc
|
||||
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
|
||||
# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
|
||||
# with the main vllm-build stage.
|
||||
FROM ubuntu:22.04 AS rust-build
|
||||
FROM base AS rust-build
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git build-essential unzip python3 python3-pip \
|
||||
build-essential unzip python3 python3-pip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY tools/install_protoc.sh /tmp/install_protoc.sh
|
||||
@@ -101,6 +145,16 @@ WORKDIR /workspace
|
||||
COPY requirements/build/rust.txt requirements/build/rust.txt
|
||||
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
|
||||
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_ENDPOINT
|
||||
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
|
||||
ARG SCCACHE_REGION_NAME=us-west-2
|
||||
|
||||
ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
|
||||
ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
|
||||
# Avoid port collision with vllm-build's own sccache daemon.
|
||||
ENV SCCACHE_SERVER_PORT=4227
|
||||
|
||||
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
|
||||
# by the wheel build stage.
|
||||
COPY rust rust
|
||||
@@ -117,10 +171,20 @@ ENV CARGO_BUILD_JOBS=4
|
||||
# cache reuse.
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
|
||||
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
|
||||
bash build_rust.sh
|
||||
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
export RUSTC_WRAPPER=sccache; \
|
||||
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
|
||||
sccache --show-stats; \
|
||||
fi && \
|
||||
bash build_rust.sh && \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
|
||||
######################### BUILD IMAGE #########################
|
||||
FROM base AS vllm-build
|
||||
FROM base-arch AS vllm-build
|
||||
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_ENDPOINT
|
||||
|
||||
ARG GIT_REPO_CHECK=0
|
||||
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
|
||||
@@ -160,17 +224,27 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=cache,target=/root/.cache/ccache \
|
||||
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
|
||||
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
|
||||
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
|
||||
sccache --show-stats; \
|
||||
fi && \
|
||||
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 && \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
|
||||
######################### TRITON-CPU BUILD IMAGE #########################
|
||||
FROM base AS vllm-triton-cpu-build
|
||||
FROM base-arch AS vllm-triton-cpu-build
|
||||
|
||||
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
|
||||
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so it
|
||||
# does not inherit the ARG/ENV defined there. Without it, the guard below would
|
||||
# see an empty value and build triton-cpu on non-x86 targets (e.g. arm64).
|
||||
# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
|
||||
# so it does not inherit the ARG/ENV defined there. Without it, the guard
|
||||
# below would see an empty value and build triton-cpu on non-x86 targets
|
||||
# (e.g. arm64).
|
||||
ARG VLLM_CPU_X86=0
|
||||
|
||||
ARG USE_SCCACHE
|
||||
ARG SCCACHE_ENDPOINT
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
|
||||
RUN mkdir dist
|
||||
@@ -178,15 +252,22 @@ RUN mkdir dist
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=cache,target=/root/.cache/ccache \
|
||||
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
|
||||
--mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \
|
||||
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
|
||||
git clone --recurse-submodules "https://github.com/triton-lang/triton-cpu.git"; \
|
||||
cd triton-cpu; \
|
||||
git checkout "270e696d"; \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \
|
||||
export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \
|
||||
sccache --show-stats; \
|
||||
fi; \
|
||||
uv build --wheel --out-dir=../dist; \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \
|
||||
fi
|
||||
|
||||
######################### TEST DEPS #########################
|
||||
FROM base AS vllm-test-deps
|
||||
FROM base-arch AS vllm-test-deps
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
|
||||
@@ -260,11 +341,11 @@ ENV HF_XET_HIGH_PERFORMANCE 1
|
||||
ENV HF_HUB_DOWNLOAD_TIMEOUT 60
|
||||
|
||||
######################### RELEASE IMAGE #########################
|
||||
FROM base AS vllm-openai
|
||||
FROM base-arch AS vllm-openai
|
||||
|
||||
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so the
|
||||
# RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 would
|
||||
# otherwise see an empty value and try to install it on non-x86 targets.
|
||||
# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`),
|
||||
# so the RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86
|
||||
# would otherwise see an empty value and try to install it on non-x86 targets.
|
||||
ARG VLLM_CPU_X86=0
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
|
||||
Reference in New Issue
Block a user