diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 90078fc0c1..22391291b4 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -15,9 +15,9 @@ set -euo pipefail DEFAULT_REPO_SLUG="vllm-project/vllm" DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl" -DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh" +DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh" DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm" -DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base" +DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_rixl build_rocshmem build_deepep mori_base ci_base" DEFAULT_CI_BASE_METADATA_VERSION="1" IMAGE_EXISTED_BEFORE_BUILD=0 @@ -764,7 +764,7 @@ configure_ci_base_image_refs() { fi set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}" set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}" - set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT:-}" + set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}" set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}" return 0 fi @@ -1211,12 +1211,24 @@ uses_rocm_csrc_cache() { esac } +uses_rocm_rust_cache() { + case "${TARGET}" in + rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm) + return 0 + ;; + *) + return 1 + ;; + esac +} + compute_rocm_csrc_content_hash() { local bake_dir="" local dockerfile_rocm="" local -a content_paths=( "requirements/common.txt" "requirements/rocm.txt" + "pyproject.toml" "setup.py" "CMakeLists.txt" "cmake" @@ -1260,6 +1272,56 @@ compute_rocm_csrc_content_hash_if_needed() { echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}" } +compute_rocm_rust_content_hash() { + local bake_dir="" + local dockerfile_rocm="" + local -a content_paths=( + "requirements/build/rust.txt" + "rust/Cargo.lock" + "rust/Cargo.toml" + "rust/proto" + "rust/src" + "rust-toolchain.toml" + "tools/build_rust.py" + "tools/install_protoc.sh" + "build_rust.sh" + ) + local -a content_args=() + + bake_dir=$(dirname "${VLLM_BAKE_FILE}") + dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + mapfile -t content_args < <( + get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}" + ) + + { + printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")" + printf 'dockerfile:%s\n' "${dockerfile_rocm}" + printf 'resolved-build-args:\n' + hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}" + printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n' + if [[ -f "${dockerfile_rocm}" ]]; then + hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" + else + printf 'missing:%s\n' "${dockerfile_rocm}" + fi + } | sha256sum | cut -d' ' -f1 +} + +compute_rocm_rust_content_hash_if_needed() { + local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}" + + if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then + return 0 + fi + + ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash) + ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}" + export ROCM_RUST_CONTENT_HASH + export ROCM_RUST_CONTENT_CACHE_REF + echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}" +} + write_hcl_string_list_entries() { local indent="$1" local value="" @@ -1317,6 +1379,7 @@ write_rocm_build_arg_override() { "${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \ "${CI_BASE_CONTENT_ARGS:-}" get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}" + get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}" } | awk 'NF && !seen[$0]++' ) @@ -1365,46 +1428,133 @@ validate_cache_export_mode() { esac } +validate_content_cache_export_mode() { + local mode="$1" + local env_name="$2" + + case "${mode}" in + missing|always|never) + ;; + *) + echo "Error: ${env_name} must be one of: missing, always, never" + exit 1 + ;; + esac +} + +should_export_content_cache_ref() { + local cache_ref="$1" + local cache_name="$2" + local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}" + + case "${mode}" in + always) + echo "${cache_name} content cache export mode is always; exporting ${cache_ref}" + return 0 + ;; + never) + echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}" + return 1 + ;; + missing|"") + if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then + echo "${cache_name} content cache exists; not re-exporting ${cache_ref}" + return 1 + fi + echo "${cache_name} content cache missing; will export ${cache_ref}" + return 0 + ;; + *) + echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never" + exit 1 + ;; + esac +} + write_rocm_cache_override() { local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}" + local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}" local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}" + local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}" local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}" - local -a content_cache_from=() + local -a csrc_content_cache_from=() + local -a rust_content_cache_from=() + local -a combined_content_cache_from=() local -a csrc_cache_to=() + local -a rust_cache_to=() local -a rocm_cache_to=() local -a export_wheel_cache_to=() + local export_csrc_cache=1 + local export_rust_cache=1 - if ! uses_rocm_csrc_cache; then + if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then return 0 fi + validate_content_cache_export_mode \ + "${content_cache_export_mode}" \ + "ROCM_CONTENT_CACHE_EXPORT_MODE" validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE" + validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE" validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE" + echo "ROCm content cache export mode: ${content_cache_export_mode}" echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}" + echo "ROCm Rust cache export mode: ${rust_cache_to_mode}" echo "ROCm final image cache export mode: ${rocm_cache_to_mode}" if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then - content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}") - csrc_cache_to+=( - "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true" - ) + csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}") + if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then + csrc_cache_to+=( + "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true" + ) + else + export_csrc_cache=0 + fi fi + if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then + rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}") + if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then + rust_cache_to+=( + "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true" + ) + else + export_rust_cache=0 + fi + fi + + combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}") + # Docker Hub cache exports are best-effort. A cache-only target failure can # otherwise cancel the sibling image target before its manifest is pushed. if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then - csrc_cache_to+=( - "type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true" - ) + if [[ ${export_csrc_cache} -eq 1 ]]; then + csrc_cache_to+=( + "type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true" + ) + fi + if [[ ${export_rust_cache} -eq 1 ]]; then + rust_cache_to+=( + "type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true" + ) + fi rocm_cache_to+=( "type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true" ) fi if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then - csrc_cache_to+=( - "type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true" - ) + if [[ ${export_csrc_cache} -eq 1 ]]; then + csrc_cache_to+=( + "type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true" + ) + fi + if [[ ${export_rust_cache} -eq 1 ]]; then + rust_cache_to+=( + "type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true" + ) + fi rocm_cache_to+=( "type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true" ) @@ -1422,7 +1572,7 @@ target "csrc-rocm-ci" { cache-from = concat( get_cache_from_rocm_csrc(), EOF - write_hcl_string_list " " "${content_cache_from[@]}" + write_hcl_string_list " " "${csrc_content_cache_from[@]}" cat </dev/null 2>&1; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain none; \ + fi \ + && rustup toolchain install "${TOOLCHAIN}" \ + && rustup default "${TOOLCHAIN}" \ + && rustc --version \ + && cargo --version + +# ----------------------- +# Rust build stage +# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages +# don't need the rust toolchain or protoc. +FROM rust-toolchain AS rust-build +ARG COMMON_WORKDIR +ARG USE_SCCACHE + # Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit # (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). ENV CARGO_BUILD_JOBS=4 ENV CARGO_NET_RETRY=10 ENV RUSTUP_MAX_RETRIES=10 +WORKDIR ${COMMON_WORKDIR}/vllm + +COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt + # BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on # a separate local sccache daemon while sharing the same remote cache backend. ENV SCCACHE_SERVER_PORT=4227 RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ - cd ${COMMON_WORKDIR}/vllm \ - && uv pip install --system -r requirements/build/rust.txt + uv pip install --system -r requirements/build/rust.txt + +COPY --from=rust-input /rust-input/vllm/rust rust +COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml +COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py +COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh # Build the release binary. Cargo's registry/git caches can be written by # concurrent BuildKit jobs on shared workers, so lock those cache mounts while @@ -159,8 +243,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ # can outlive source updates across BuildKit cache reuse. RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \ --mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \ - cd ${COMMON_WORKDIR}/vllm \ - && if [ "$USE_SCCACHE" = "1" ]; then \ + if [ "$USE_SCCACHE" = "1" ]; then \ export RUSTC_WRAPPER=sccache \ && sccache --show-stats; \ fi \ @@ -185,8 +268,8 @@ COPY requirements/common.txt requirements/common.txt RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ uv pip install --system -r requirements/rocm.txt -# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do -# not invalidate the expensive native build layer. +# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image, +# but it still participates in the BuildKit cache key for this RUN step. COPY setup.py CMakeLists.txt ./ COPY tools/build_rust.py tools/build_rust.py COPY cmake cmake/ @@ -556,6 +639,17 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1 FROM mori_base AS ci_base ARG COMMON_WORKDIR +ENV CARGO_HOME=/root/.cargo +ENV RUSTUP_HOME=/root/.rustup +ENV PATH=${CARGO_HOME}/bin:${PATH} + +# Bake the Rust/protoc toolchain into ci_base so Rust build/test steps do not +# need to fetch it on each per-commit image build. +COPY --from=rust-toolchain /root/.cargo /root/.cargo +COPY --from=rust-toolchain /root/.rustup /root/.rustup +COPY --from=rust-toolchain /usr/local/bin/protoc /usr/local/bin/protoc +COPY --from=rust-toolchain /usr/local/include/google /usr/local/include/google + # Update rdma-core to support latest rocshmem. ARG DEEPEP_NIC RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \ diff --git a/docker/ci-rocm.hcl b/docker/ci-rocm.hcl index f06e071c93..0ae991bd52 100644 --- a/docker/ci-rocm.hcl +++ b/docker/ci-rocm.hcl @@ -141,6 +141,10 @@ variable "ROCM_CSRC_CACHE_TO_MODE" { default = "max" } +variable "ROCM_RUST_CACHE_TO_MODE" { + default = "max" +} + variable "ROCM_FINAL_CACHE_TO_MODE" { default = "min" } @@ -164,6 +168,13 @@ function "get_cache_from_rocm" { VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "", ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "", ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "", + # Import the source-scoped Rust frontend cache so non-Rust changes do not + # force a fresh cargo release build. + BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "", + PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "", + VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "", + ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "", + ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "", # Branch-scoped full image cache - fallback when parent-commit cache is evicted ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "", ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "", @@ -204,6 +215,27 @@ function "get_cache_to_rocm_csrc" { ]) } +function "get_cache_from_rocm_rust" { + params = [] + result = compact([ + BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "", + PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "", + VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "", + ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "", + ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "", + ]) +} + +function "get_cache_to_rocm_rust" { + params = [] + result = compact([ + # Export exact-commit and branch-scoped Rust caches. A content-addressed + # cache ref is appended by ci-bake-rocm.sh when that wrapper is used. + BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT},mode=${ROCM_RUST_CACHE_TO_MODE}" : "", + ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${ROCM_RUST_CACHE_TO_MODE}" : "", + ]) +} + # Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP). # These stages are pinned to specific upstream commit hashes, so cache keys use # those hashes rather than the Buildkite commit. This means the cache persists @@ -276,6 +308,17 @@ target "csrc-rocm-ci" { output = ["type=cacheonly"] } +# Cache-only target for the Rust frontend build stage. Final-image cache +# exports use mode=min and do not reliably persist intermediate cargo layers, +# so Rust gets its own source-scoped cache target. +target "rust-rocm-ci" { + inherits = ["_common-rocm", "_ci-rocm"] + target = "rust-build" + cache-from = get_cache_from_rocm_rust() + cache-to = get_cache_to_rocm_rust() + output = ["type=cacheonly"] +} + # Keep wheel export on the same CI graph as the test image build so the # shared build_vllm/export_vllm stages resolve identically within one bake # invocation. Without this, export-wheel-rocm uses the plain local target @@ -292,13 +335,13 @@ target "export-wheel-rocm" { # Artifact-only vLLM build. GPU test jobs consume this artifact on top of # ci_base, avoiding a per-commit multi-GB image push/pull. group "test-rocm-ci-with-artifacts" { - targets = ["csrc-rocm-ci", "export-wheel-rocm"] + targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"] } # Full test image + wheel export. Kept for fallback/debugging when a pushed # per-commit image is useful. group "test-rocm-ci-with-wheel" { - targets = ["csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"] + targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"] } # Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG