[ROCm][CI] Cache Rust builds by source inputs (#46527)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-13 01:14:08 -05:00
committed by GitHub
co-authored by Matthew Wong
parent 36484e464a
commit b7b58d1eba
3 changed files with 337 additions and 37 deletions
+111 -17
View File
@@ -1,5 +1,7 @@
# default base image
ARG REMOTE_VLLM="0"
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
ARG VLLM_BRANCH="main"
ARG COMMON_WORKDIR=/app
ARG BASE_IMAGE=rocm/vllm-dev:base
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
@@ -110,10 +112,8 @@ WORKDIR ${COMMON_WORKDIR}
FROM base AS fetch_vllm_0
ONBUILD COPY ./ vllm/
FROM base AS fetch_vllm_1
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
ARG VLLM_BRANCH="main"
ENV VLLM_REPO=${VLLM_REPO}
ENV VLLM_BRANCH=${VLLM_BRANCH}
ARG VLLM_REPO
ARG VLLM_BRANCH
ONBUILD RUN git clone ${VLLM_REPO} \
&& cd vllm \
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
@@ -124,34 +124,118 @@ ONBUILD RUN git clone ${VLLM_REPO} \
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
# -----------------------
# Rust build stage
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
# don't need the rust toolchain or protoc.
FROM fetch_vllm AS rust-build
ARG COMMON_WORKDIR
ARG USE_SCCACHE
# Rust/protoc toolchain inputs
#
# Keep these separate from Rust source inputs so Rust source-only changes do not
# invalidate the long-lived ci_base image that carries the toolchain.
FROM scratch AS rust_toolchain_input_0
COPY rust-toolchain.toml /rust-toolchain-input/vllm/rust-toolchain.toml
COPY tools/install_protoc.sh /rust-toolchain-input/vllm/tools/install_protoc.sh
FROM base AS rust_toolchain_input_1
ARG VLLM_REPO
ARG VLLM_BRANCH
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-toolchain-input/vllm \
&& cd /rust-toolchain-input/vllm \
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
&& git sparse-checkout init --no-cone \
&& git sparse-checkout set \
rust-toolchain.toml \
tools/install_protoc.sh \
&& git checkout FETCH_HEAD
FROM rust_toolchain_input_${REMOTE_VLLM} AS rust-toolchain-input
# -----------------------
# Rust build inputs
#
# REMOTE_VLLM=0: use only the local files that can affect Rust artifacts.
# REMOTE_VLLM=1: fetch only those same files from the requested remote ref.
FROM scratch AS rust_input_0
COPY requirements/build/rust.txt /rust-input/vllm/requirements/build/rust.txt
COPY rust/Cargo.lock rust/Cargo.toml /rust-input/vllm/rust/
COPY rust/proto /rust-input/vllm/rust/proto
COPY rust/src /rust-input/vllm/rust/src
COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
COPY build_rust.sh /rust-input/vllm/build_rust.sh
FROM base AS rust_input_1
ARG VLLM_REPO
ARG VLLM_BRANCH
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
&& cd /rust-input/vllm \
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
&& git sparse-checkout init --no-cone \
&& git sparse-checkout set \
requirements/build/rust.txt \
rust/Cargo.lock \
rust/Cargo.toml \
rust/proto/** \
rust/src/** \
rust-toolchain.toml \
tools/build_rust.py \
build_rust.sh \
&& git checkout FETCH_HEAD
FROM rust_input_${REMOTE_VLLM} AS rust-input
# -----------------------
# Rust/protoc toolchain
FROM base AS rust-toolchain
ENV CARGO_HOME=/root/.cargo
ENV RUSTUP_HOME=/root/.rustup
ENV PATH=${CARGO_HOME}/bin:${PATH}
# protoc is used by tonic-build/prost-build.
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
ca-certificates curl unzip \
&& rm -rf /var/lib/apt/lists/*
COPY tools/install_protoc.sh /tmp/install_protoc.sh
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/tools/install_protoc.sh /tmp/install_protoc.sh
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/rust-toolchain.toml /tmp/rust-toolchain.toml
RUN TOOLCHAIN="$(grep '^channel' /tmp/rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/')" \
&& if ! command -v rustup >/dev/null 2>&1; then \
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --default-toolchain none; \
fi \
&& rustup toolchain install "${TOOLCHAIN}" \
&& rustup default "${TOOLCHAIN}" \
&& rustc --version \
&& cargo --version
# -----------------------
# Rust build stage
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
# don't need the rust toolchain or protoc.
FROM rust-toolchain AS rust-build
ARG COMMON_WORKDIR
ARG USE_SCCACHE
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
ENV CARGO_BUILD_JOBS=4
ENV CARGO_NET_RETRY=10
ENV RUSTUP_MAX_RETRIES=10
WORKDIR ${COMMON_WORKDIR}/vllm
COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt
# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
# a separate local sccache daemon while sharing the same remote cache backend.
ENV SCCACHE_SERVER_PORT=4227
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
cd ${COMMON_WORKDIR}/vllm \
&& uv pip install --system -r requirements/build/rust.txt
uv pip install --system -r requirements/build/rust.txt
COPY --from=rust-input /rust-input/vllm/rust rust
COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
# Build the release binary. Cargo's registry/git caches can be written by
# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
@@ -159,8 +243,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
# can outlive source updates across BuildKit cache reuse.
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
cd ${COMMON_WORKDIR}/vllm \
&& if [ "$USE_SCCACHE" = "1" ]; then \
if [ "$USE_SCCACHE" = "1" ]; then \
export RUSTC_WRAPPER=sccache \
&& sccache --show-stats; \
fi \
@@ -185,8 +268,8 @@ COPY requirements/common.txt requirements/common.txt
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
uv pip install --system -r requirements/rocm.txt
# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do
# not invalidate the expensive native build layer.
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
# but it still participates in the BuildKit cache key for this RUN step.
COPY setup.py CMakeLists.txt ./
COPY tools/build_rust.py tools/build_rust.py
COPY cmake cmake/
@@ -556,6 +639,17 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
FROM mori_base AS ci_base
ARG COMMON_WORKDIR
ENV CARGO_HOME=/root/.cargo
ENV RUSTUP_HOME=/root/.rustup
ENV PATH=${CARGO_HOME}/bin:${PATH}
# Bake the Rust/protoc toolchain into ci_base so Rust build/test steps do not
# need to fetch it on each per-commit image build.
COPY --from=rust-toolchain /root/.cargo /root/.cargo
COPY --from=rust-toolchain /root/.rustup /root/.rustup
COPY --from=rust-toolchain /usr/local/bin/protoc /usr/local/bin/protoc
COPY --from=rust-toolchain /usr/local/include/google /usr/local/include/google
# Update rdma-core to support latest rocshmem.
ARG DEEPEP_NIC
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
+45 -2
View File
@@ -141,6 +141,10 @@ variable "ROCM_CSRC_CACHE_TO_MODE" {
default = "max"
}
variable "ROCM_RUST_CACHE_TO_MODE" {
default = "max"
}
variable "ROCM_FINAL_CACHE_TO_MODE" {
default = "min"
}
@@ -164,6 +168,13 @@ function "get_cache_from_rocm" {
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
# Import the source-scoped Rust frontend cache so non-Rust changes do not
# force a fresh cargo release build.
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
# Branch-scoped full image cache - fallback when parent-commit cache is evicted
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
@@ -204,6 +215,27 @@ function "get_cache_to_rocm_csrc" {
])
}
function "get_cache_from_rocm_rust" {
params = []
result = compact([
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
])
}
function "get_cache_to_rocm_rust" {
params = []
result = compact([
# Export exact-commit and branch-scoped Rust caches. A content-addressed
# cache ref is appended by ci-bake-rocm.sh when that wrapper is used.
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
])
}
# Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP).
# These stages are pinned to specific upstream commit hashes, so cache keys use
# those hashes rather than the Buildkite commit. This means the cache persists
@@ -276,6 +308,17 @@ target "csrc-rocm-ci" {
output = ["type=cacheonly"]
}
# Cache-only target for the Rust frontend build stage. Final-image cache
# exports use mode=min and do not reliably persist intermediate cargo layers,
# so Rust gets its own source-scoped cache target.
target "rust-rocm-ci" {
inherits = ["_common-rocm", "_ci-rocm"]
target = "rust-build"
cache-from = get_cache_from_rocm_rust()
cache-to = get_cache_to_rocm_rust()
output = ["type=cacheonly"]
}
# Keep wheel export on the same CI graph as the test image build so the
# shared build_vllm/export_vllm stages resolve identically within one bake
# invocation. Without this, export-wheel-rocm uses the plain local target
@@ -292,13 +335,13 @@ target "export-wheel-rocm" {
# Artifact-only vLLM build. GPU test jobs consume this artifact on top of
# ci_base, avoiding a per-commit multi-GB image push/pull.
group "test-rocm-ci-with-artifacts" {
targets = ["csrc-rocm-ci", "export-wheel-rocm"]
targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
}
# Full test image + wheel export. Kept for fallback/debugging when a pushed
# per-commit image is useful.
group "test-rocm-ci-with-wheel" {
targets = ["csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
}
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG