diff --git a/.buildkite/image_build/image_build_cpu.sh b/.buildkite/image_build/image_build_cpu.sh index 3ac1a110f6..573c4564f1 100755 --- a/.buildkite/image_build/image_build_cpu.sh +++ b/.buildkite/image_build/image_build_cpu.sh @@ -11,24 +11,91 @@ REPO=$2 BUILDKITE_COMMIT=$3 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" -# authenticate with AWS ECR +# replace invalid characters in Docker image tags and truncate to 128 chars +clean_docker_tag() { + local input="$1" + echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 +} + +# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN +# Reuses the same ECR cache repos as the CUDA image build, with an +# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated. +prepare_cache_tags() { + TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache" + MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache" + + if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then + if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then + cache="${MAIN_CACHE_ECR}:latest-x86_cpu" + else + clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}") + cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu" + fi + CACHE_TO="$cache" + CACHE_FROM="$cache" + CACHE_FROM_BASE_BRANCH="$cache" + else + CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu" + CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu" + if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then + CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu" + else + clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}") + CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu" + fi + fi + + CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu" +} + +# authenticate with AWS ECR (public, for the image; private, for the cache repos) aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true # skip build if image already exists if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then echo "Image found" else echo "Image not found, proceeding with build..." - # build - docker build --file docker/Dockerfile.cpu \ + + prepare_cache_tags + echo "--- :mag: Cache tags" + echo "CACHE_TO: ${CACHE_TO}" + echo "CACHE_FROM: ${CACHE_FROM}" + echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}" + echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}" + + # dedupe cache-from refs so identical branch/main fallbacks aren't repeated + CACHE_FROM_ARGS=() + SEEN_REFS="" + for ref in "$CACHE_FROM" "$CACHE_FROM_BASE_BRANCH" "$CACHE_FROM_MAIN"; do + if [[ "$SEEN_REFS" != *"|${ref}|"* ]]; then + CACHE_FROM_ARGS+=(--cache-from "type=registry,ref=${ref}") + SEEN_REFS="${SEEN_REFS}|${ref}|" + fi + done + + echo "--- :docker: Setting up buildx" + # network=host so the isolated docker-container builder can still reach the + # EC2 instance metadata service for sccache's S3 credentials; recreate on + # every run in case a stale builder without this driver-opt already exists + # on a warm CI host. + docker buildx rm vllm-cpu-builder >/dev/null 2>&1 || true + docker buildx create --name vllm-cpu-builder --driver docker-container --driver-opt network=host --use + docker buildx inspect --bootstrap + + # build and push + docker buildx build --file docker/Dockerfile.cpu \ --build-arg max_jobs=16 \ --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ --build-arg VLLM_CPU_X86=true \ + --build-arg USE_SCCACHE=1 \ --tag "$IMAGE" \ --target vllm-test \ + "${CACHE_FROM_ARGS[@]}" \ + --cache-to "type=registry,ref=${CACHE_TO},mode=max" \ + --push \ --progress plain . - # push - docker push "$IMAGE" fi .buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index a528ffbd8d..9e86ed1499 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -19,8 +19,42 @@ # VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation) # +######################### BASE IMAGE ######################### +# Common apt packages and the optional sccache binary install, shared by +# base-common and rust-build (the latter is deliberately not `FROM +# base-common`, to stay minimal and build in parallel with vllm-build). +# +# Optional remote (S3-backed) compiler cache. Local BuildKit `--mount=type=cache` +# cache mounts (ccache, cargo registry) aren't exported by `--cache-to +# type=registry`, so they don't survive a build landing on a different/fresh +# builder. sccache's cache lives in S3 instead, so it does. +FROM ubuntu:22.04 AS base + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -y \ + && apt-get install -y --no-install-recommends ca-certificates curl git + +ARG TARGETARCH +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL + +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + echo "Installing sccache..." \ + && case "${TARGETARCH}" in \ + arm64) SCCACHE_ARCH="aarch64" ;; \ + amd64) SCCACHE_ARCH="x86_64" ;; \ + *) echo "Unsupported TARGETARCH for sccache: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && export SCCACHE_DOWNLOAD_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ + && curl -L -o sccache.tar.gz "${SCCACHE_DOWNLOAD_URL}" \ + && tar -xzf sccache.tar.gz \ + && mv sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ + && rm -rf sccache.tar.gz sccache-v0.8.1-${SCCACHE_ARCH}-unknown-linux-musl; \ + fi + ######################### COMMON BASE IMAGE ######################### -FROM ubuntu:22.04 AS base-common +FROM base AS base-common WORKDIR /workspace @@ -33,11 +67,22 @@ ENV MAX_JOBS=${max_jobs} RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -y \ - && apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates zlib1g-dev \ + && apt-get install -y --no-install-recommends sudo ccache wget zlib1g-dev \ gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof make xz-utils \ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ && curl -LsSf https://astral.sh/uv/install.sh | sh +ARG TARGETARCH +ARG USE_SCCACHE +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION_NAME=us-west-2 +ARG SCCACHE_S3_NO_CREDENTIALS=0 + +ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} +ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} +ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} +ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} + # Compiler and linker environment ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12 ENV CCACHE_DIR=/root/.cache/ccache @@ -63,7 +108,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade pip && \ uv pip install -r requirements/cpu.txt --torch-backend cpu -ARG TARGETARCH ENV TARGETARCH=${TARGETARCH} ######################### x86_64 BASE IMAGE ######################### @@ -76,8 +120,8 @@ FROM base-common AS base-arm64 ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4" -######################### BASE IMAGE ######################### -FROM base-${TARGETARCH} AS base +######################### ARCH BASE IMAGE ######################### +FROM base-${TARGETARCH} AS base-arch RUN echo 'ulimit -c 0' >> ~/.bashrc @@ -85,12 +129,12 @@ RUN echo 'ulimit -c 0' >> ~/.bashrc # Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build # stage doesn't need the rust toolchain or protoc. This stage runs in parallel # with the main vllm-build stage. -FROM ubuntu:22.04 AS rust-build +FROM base AS rust-build ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y \ && apt-get install -y --no-install-recommends \ - ca-certificates curl git build-essential unzip python3 python3-pip \ + build-essential unzip python3 python3-pip \ && rm -rf /var/lib/apt/lists/* COPY tools/install_protoc.sh /tmp/install_protoc.sh @@ -101,6 +145,16 @@ WORKDIR /workspace COPY requirements/build/rust.txt requirements/build/rust.txt RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt +ARG USE_SCCACHE +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION_NAME=us-west-2 + +ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} +ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} +# Avoid port collision with vllm-build's own sccache daemon. +ENV SCCACHE_SERVER_PORT=4227 + # Copy only the Rust build inputs; build_rust.sh publishes artifacts needed # by the wheel build stage. COPY rust rust @@ -117,10 +171,20 @@ ENV CARGO_BUILD_JOBS=4 # cache reuse. RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,target=/root/.cargo/git,sharing=locked \ - bash build_rust.sh + --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ + if [ "$USE_SCCACHE" = "1" ]; then \ + export RUSTC_WRAPPER=sccache; \ + if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \ + sccache --show-stats; \ + fi && \ + bash build_rust.sh && \ + if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi ######################### BUILD IMAGE ######################### -FROM base AS vllm-build +FROM base-arch AS vllm-build + +ARG USE_SCCACHE +ARG SCCACHE_ENDPOINT ARG GIT_REPO_CHECK=0 # Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ... @@ -160,17 +224,27 @@ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/root/.cache/ccache \ --mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \ - VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 + --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ + if [ "$USE_SCCACHE" = "1" ]; then \ + if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \ + sccache --show-stats; \ + fi && \ + VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 && \ + if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi ######################### TRITON-CPU BUILD IMAGE ######################### -FROM base AS vllm-triton-cpu-build +FROM base-arch AS vllm-triton-cpu-build # Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ... -# Re-declared here because this stage is `FROM base` (not `vllm-build`), so it -# does not inherit the ARG/ENV defined there. Without it, the guard below would -# see an empty value and build triton-cpu on non-x86 targets (e.g. arm64). +# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`), +# so it does not inherit the ARG/ENV defined there. Without it, the guard +# below would see an empty value and build triton-cpu on non-x86 targets +# (e.g. arm64). ARG VLLM_CPU_X86=0 +ARG USE_SCCACHE +ARG SCCACHE_ENDPOINT + WORKDIR /vllm-workspace RUN mkdir dist @@ -178,15 +252,22 @@ RUN mkdir dist RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/root/.cache/ccache \ --mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \ + --mount=type=secret,id=aws-credentials,target=/root/.aws/credentials,required=false \ if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \ git clone --recurse-submodules "https://github.com/triton-lang/triton-cpu.git"; \ cd triton-cpu; \ git checkout "270e696d"; \ + if [ "$USE_SCCACHE" = "1" ]; then \ + if [ -n "${SCCACHE_ENDPOINT}" ]; then export SCCACHE_ENDPOINT="${SCCACHE_ENDPOINT}"; fi; \ + export TRITON_APPEND_CMAKE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"; \ + sccache --show-stats; \ + fi; \ uv build --wheel --out-dir=../dist; \ + if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi; \ fi ######################### TEST DEPS ######################### -FROM base AS vllm-test-deps +FROM base-arch AS vllm-test-deps WORKDIR /vllm-workspace @@ -260,11 +341,11 @@ ENV HF_XET_HIGH_PERFORMANCE 1 ENV HF_HUB_DOWNLOAD_TIMEOUT 60 ######################### RELEASE IMAGE ######################### -FROM base AS vllm-openai +FROM base-arch AS vllm-openai -# Re-declared here because this stage is `FROM base` (not `vllm-build`), so the -# RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 would -# otherwise see an empty value and try to install it on non-x86 targets. +# Re-declared here because this stage is `FROM base-arch` (not `vllm-build`), +# so the RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 +# would otherwise see an empty value and try to install it on non-x86 targets. ARG VLLM_CPU_X86=0 WORKDIR /vllm-workspace