[ROCm][CI] Cache Rust builds by source inputs (#46527)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
co-authored by
Matthew Wong
parent
36484e464a
commit
b7b58d1eba
+111
-17
@@ -1,5 +1,7 @@
|
||||
# default base image
|
||||
ARG REMOTE_VLLM="0"
|
||||
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
|
||||
ARG VLLM_BRANCH="main"
|
||||
ARG COMMON_WORKDIR=/app
|
||||
ARG BASE_IMAGE=rocm/vllm-dev:base
|
||||
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
|
||||
@@ -110,10 +112,8 @@ WORKDIR ${COMMON_WORKDIR}
|
||||
FROM base AS fetch_vllm_0
|
||||
ONBUILD COPY ./ vllm/
|
||||
FROM base AS fetch_vllm_1
|
||||
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
|
||||
ARG VLLM_BRANCH="main"
|
||||
ENV VLLM_REPO=${VLLM_REPO}
|
||||
ENV VLLM_BRANCH=${VLLM_BRANCH}
|
||||
ARG VLLM_REPO
|
||||
ARG VLLM_BRANCH
|
||||
ONBUILD RUN git clone ${VLLM_REPO} \
|
||||
&& cd vllm \
|
||||
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||
@@ -124,34 +124,118 @@ ONBUILD RUN git clone ${VLLM_REPO} \
|
||||
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
|
||||
|
||||
# -----------------------
|
||||
# Rust build stage
|
||||
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
||||
# don't need the rust toolchain or protoc.
|
||||
FROM fetch_vllm AS rust-build
|
||||
ARG COMMON_WORKDIR
|
||||
ARG USE_SCCACHE
|
||||
# Rust/protoc toolchain inputs
|
||||
#
|
||||
# Keep these separate from Rust source inputs so Rust source-only changes do not
|
||||
# invalidate the long-lived ci_base image that carries the toolchain.
|
||||
FROM scratch AS rust_toolchain_input_0
|
||||
COPY rust-toolchain.toml /rust-toolchain-input/vllm/rust-toolchain.toml
|
||||
COPY tools/install_protoc.sh /rust-toolchain-input/vllm/tools/install_protoc.sh
|
||||
|
||||
FROM base AS rust_toolchain_input_1
|
||||
ARG VLLM_REPO
|
||||
ARG VLLM_BRANCH
|
||||
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-toolchain-input/vllm \
|
||||
&& cd /rust-toolchain-input/vllm \
|
||||
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||
&& git sparse-checkout init --no-cone \
|
||||
&& git sparse-checkout set \
|
||||
rust-toolchain.toml \
|
||||
tools/install_protoc.sh \
|
||||
&& git checkout FETCH_HEAD
|
||||
|
||||
FROM rust_toolchain_input_${REMOTE_VLLM} AS rust-toolchain-input
|
||||
|
||||
# -----------------------
|
||||
# Rust build inputs
|
||||
#
|
||||
# REMOTE_VLLM=0: use only the local files that can affect Rust artifacts.
|
||||
# REMOTE_VLLM=1: fetch only those same files from the requested remote ref.
|
||||
FROM scratch AS rust_input_0
|
||||
COPY requirements/build/rust.txt /rust-input/vllm/requirements/build/rust.txt
|
||||
COPY rust/Cargo.lock rust/Cargo.toml /rust-input/vllm/rust/
|
||||
COPY rust/proto /rust-input/vllm/rust/proto
|
||||
COPY rust/src /rust-input/vllm/rust/src
|
||||
COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
|
||||
COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
|
||||
COPY build_rust.sh /rust-input/vllm/build_rust.sh
|
||||
|
||||
FROM base AS rust_input_1
|
||||
ARG VLLM_REPO
|
||||
ARG VLLM_BRANCH
|
||||
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
|
||||
&& cd /rust-input/vllm \
|
||||
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||
&& git sparse-checkout init --no-cone \
|
||||
&& git sparse-checkout set \
|
||||
requirements/build/rust.txt \
|
||||
rust/Cargo.lock \
|
||||
rust/Cargo.toml \
|
||||
rust/proto/** \
|
||||
rust/src/** \
|
||||
rust-toolchain.toml \
|
||||
tools/build_rust.py \
|
||||
build_rust.sh \
|
||||
&& git checkout FETCH_HEAD
|
||||
|
||||
FROM rust_input_${REMOTE_VLLM} AS rust-input
|
||||
|
||||
# -----------------------
|
||||
# Rust/protoc toolchain
|
||||
FROM base AS rust-toolchain
|
||||
|
||||
ENV CARGO_HOME=/root/.cargo
|
||||
ENV RUSTUP_HOME=/root/.rustup
|
||||
ENV PATH=${CARGO_HOME}/bin:${PATH}
|
||||
|
||||
# protoc is used by tonic-build/prost-build.
|
||||
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
|
||||
ca-certificates curl unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY tools/install_protoc.sh /tmp/install_protoc.sh
|
||||
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/tools/install_protoc.sh /tmp/install_protoc.sh
|
||||
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
|
||||
|
||||
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/rust-toolchain.toml /tmp/rust-toolchain.toml
|
||||
RUN TOOLCHAIN="$(grep '^channel' /tmp/rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/')" \
|
||||
&& if ! command -v rustup >/dev/null 2>&1; then \
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||
| sh -s -- -y --default-toolchain none; \
|
||||
fi \
|
||||
&& rustup toolchain install "${TOOLCHAIN}" \
|
||||
&& rustup default "${TOOLCHAIN}" \
|
||||
&& rustc --version \
|
||||
&& cargo --version
|
||||
|
||||
# -----------------------
|
||||
# Rust build stage
|
||||
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
||||
# don't need the rust toolchain or protoc.
|
||||
FROM rust-toolchain AS rust-build
|
||||
ARG COMMON_WORKDIR
|
||||
ARG USE_SCCACHE
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
ENV CARGO_NET_RETRY=10
|
||||
ENV RUSTUP_MAX_RETRIES=10
|
||||
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt
|
||||
|
||||
# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
|
||||
# a separate local sccache daemon while sharing the same remote cache backend.
|
||||
ENV SCCACHE_SERVER_PORT=4227
|
||||
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
cd ${COMMON_WORKDIR}/vllm \
|
||||
&& uv pip install --system -r requirements/build/rust.txt
|
||||
uv pip install --system -r requirements/build/rust.txt
|
||||
|
||||
COPY --from=rust-input /rust-input/vllm/rust rust
|
||||
COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
|
||||
COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
|
||||
COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
|
||||
|
||||
# Build the release binary. Cargo's registry/git caches can be written by
|
||||
# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
|
||||
@@ -159,8 +243,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
# can outlive source updates across BuildKit cache reuse.
|
||||
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
|
||||
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
|
||||
cd ${COMMON_WORKDIR}/vllm \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
export RUSTC_WRAPPER=sccache \
|
||||
&& sccache --show-stats; \
|
||||
fi \
|
||||
@@ -185,8 +268,8 @@ COPY requirements/common.txt requirements/common.txt
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do
|
||||
# not invalidate the expensive native build layer.
|
||||
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
|
||||
# but it still participates in the BuildKit cache key for this RUN step.
|
||||
COPY setup.py CMakeLists.txt ./
|
||||
COPY tools/build_rust.py tools/build_rust.py
|
||||
COPY cmake cmake/
|
||||
@@ -556,6 +639,17 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
|
||||
FROM mori_base AS ci_base
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
ENV CARGO_HOME=/root/.cargo
|
||||
ENV RUSTUP_HOME=/root/.rustup
|
||||
ENV PATH=${CARGO_HOME}/bin:${PATH}
|
||||
|
||||
# Bake the Rust/protoc toolchain into ci_base so Rust build/test steps do not
|
||||
# need to fetch it on each per-commit image build.
|
||||
COPY --from=rust-toolchain /root/.cargo /root/.cargo
|
||||
COPY --from=rust-toolchain /root/.rustup /root/.rustup
|
||||
COPY --from=rust-toolchain /usr/local/bin/protoc /usr/local/bin/protoc
|
||||
COPY --from=rust-toolchain /usr/local/include/google /usr/local/include/google
|
||||
|
||||
# Update rdma-core to support latest rocshmem.
|
||||
ARG DEEPEP_NIC
|
||||
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
|
||||
|
||||
+45
-2
@@ -141,6 +141,10 @@ variable "ROCM_CSRC_CACHE_TO_MODE" {
|
||||
default = "max"
|
||||
}
|
||||
|
||||
variable "ROCM_RUST_CACHE_TO_MODE" {
|
||||
default = "max"
|
||||
}
|
||||
|
||||
variable "ROCM_FINAL_CACHE_TO_MODE" {
|
||||
default = "min"
|
||||
}
|
||||
@@ -164,6 +168,13 @@ function "get_cache_from_rocm" {
|
||||
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||
# Import the source-scoped Rust frontend cache so non-Rust changes do not
|
||||
# force a fresh cargo release build.
|
||||
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
|
||||
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
|
||||
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||
# Branch-scoped full image cache - fallback when parent-commit cache is evicted
|
||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||
@@ -204,6 +215,27 @@ function "get_cache_to_rocm_csrc" {
|
||||
])
|
||||
}
|
||||
|
||||
function "get_cache_from_rocm_rust" {
|
||||
params = []
|
||||
result = compact([
|
||||
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
|
||||
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
|
||||
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||
])
|
||||
}
|
||||
|
||||
function "get_cache_to_rocm_rust" {
|
||||
params = []
|
||||
result = compact([
|
||||
# Export exact-commit and branch-scoped Rust caches. A content-addressed
|
||||
# cache ref is appended by ci-bake-rocm.sh when that wrapper is used.
|
||||
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
|
||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
|
||||
])
|
||||
}
|
||||
|
||||
# Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP).
|
||||
# These stages are pinned to specific upstream commit hashes, so cache keys use
|
||||
# those hashes rather than the Buildkite commit. This means the cache persists
|
||||
@@ -276,6 +308,17 @@ target "csrc-rocm-ci" {
|
||||
output = ["type=cacheonly"]
|
||||
}
|
||||
|
||||
# Cache-only target for the Rust frontend build stage. Final-image cache
|
||||
# exports use mode=min and do not reliably persist intermediate cargo layers,
|
||||
# so Rust gets its own source-scoped cache target.
|
||||
target "rust-rocm-ci" {
|
||||
inherits = ["_common-rocm", "_ci-rocm"]
|
||||
target = "rust-build"
|
||||
cache-from = get_cache_from_rocm_rust()
|
||||
cache-to = get_cache_to_rocm_rust()
|
||||
output = ["type=cacheonly"]
|
||||
}
|
||||
|
||||
# Keep wheel export on the same CI graph as the test image build so the
|
||||
# shared build_vllm/export_vllm stages resolve identically within one bake
|
||||
# invocation. Without this, export-wheel-rocm uses the plain local target
|
||||
@@ -292,13 +335,13 @@ target "export-wheel-rocm" {
|
||||
# Artifact-only vLLM build. GPU test jobs consume this artifact on top of
|
||||
# ci_base, avoiding a per-commit multi-GB image push/pull.
|
||||
group "test-rocm-ci-with-artifacts" {
|
||||
targets = ["csrc-rocm-ci", "export-wheel-rocm"]
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed
|
||||
# per-commit image is useful.
|
||||
group "test-rocm-ci-with-wheel" {
|
||||
targets = ["csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
|
||||
|
||||
Reference in New Issue
Block a user