From 419b51b38517bd446c6e359d37b1ee8674cd7e84 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 10 Aug 2026 21:51:39 -0500 Subject: [PATCH] [Bugfix][ROCm][CI] Stabilize build context and source caches (#51721) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas --- .buildkite/scripts/ci-bake-rocm.sh | 370 ++++++++++++++++++++++++----- docker/Dockerfile.rocm | 36 ++- 2 files changed, 345 insertions(+), 61 deletions(-) diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index b0f32cd56f..8ba73b536a 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -22,7 +22,7 @@ DEFAULT_CI_BASE_METADATA_VERSION="3" # ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected # local-source stages rather than unreachable remote-fetch alternatives. DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py" -DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies csrc-build" +DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies rocm-triton-kernels csrc-build" DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/install_protoc.sh build_rust.sh" DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build" # Docker's 128-character tag limit minus the longest cache prefix @@ -36,9 +36,14 @@ CI_HCL_PATH="" CI_BASE_LABEL_OVERRIDE_PATH="" CSRC_CACHE_OVERRIDE_PATH="" ROCM_ARG_OVERRIDE_PATH="" +BUILD_CONTEXT_OVERRIDE_PATH="" SCRIPT_TMP_DIR="" BAKE_CONFIG_FILE="" +ROCM_BUILD_CONTEXT_ROOT="" +ROCM_BUILD_CONTEXT_INDEX="" +ROCM_BUILD_CONTEXT_COMMIT="" BAKE_FILES=() +BAKE_ALLOW_ARGS=() BAKE_TARGETS=() DEPENDENCY_CACHE_TARGETS=() @@ -192,16 +197,20 @@ get_buildkite_target_repo_url() { printf 'https://github.com/%s.git\n' "${DEFAULT_REPO_SLUG}" } -git_fetch_for_cache() { +git_fetch_with_timeout() { local timeout_secs="${ROCM_CACHE_GIT_FETCH_TIMEOUT:-60}" if command -v timeout >/dev/null 2>&1; then - timeout "${timeout_secs}s" git fetch "$@" 2>/dev/null + timeout "${timeout_secs}s" git fetch "$@" else - git fetch "$@" 2>/dev/null + git fetch "$@" fi } +git_fetch_for_cache() { + git_fetch_with_timeout "$@" 2>/dev/null +} + hash_string_short() { printf '%s' "$1" | sha256sum | cut -c1-16 } @@ -209,23 +218,51 @@ hash_string_short() { list_content_files() { # Hash the checkout inputs Docker can intentionally consume, not ignored # compiler/test debris left behind on a reused worker. - git ls-files -z --cached --others --exclude-standard -- "$1" \ - | LC_ALL=C sort -z + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]]; then + GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \ + git ls-files -z --cached -- "$1" | LC_ALL=C sort -z + else + git ls-files -z --cached --others --exclude-standard -- "$1" \ + | LC_ALL=C sort -z + fi +} + +content_regular_file() { + local file="$1" + local physical_file="" + + physical_file="${file}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then + physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}" + fi + [[ -f "${physical_file}" ]] } hash_content_file() { local file="$1" + local physical_file="" + local checksum="" + local file_mode="" - if [[ -L "${file}" ]]; then - printf 'symlink:%s\ntarget:%s\n' "${file}" "$(readlink "${file}")" + physical_file="${file}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then + physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}" + fi + if [[ -L "${physical_file}" ]]; then + printf 'symlink:%s\ntarget:' "${file}" + readlink -n -- "${physical_file}" || return $? + printf '\n' return fi - if [[ ! -f "${file}" ]]; then + if [[ ! -f "${physical_file}" ]]; then printf 'missing:%s\n' "${file}" return fi - printf 'file:%s\nmode:%s\n' "${file}" "$(stat -c '%a' "${file}")" - sha256sum "${file}" + file_mode=$(stat -c '%a' "${physical_file}") || return $? + printf 'file:%s\nmode:%s\n' "${file}" "${file_mode}" + checksum=$(sha256sum < "${physical_file}") || return $? + checksum="${checksum%% *}" + printf '%s %s\n' "${checksum}" "${file}" } hash_content_directory() { @@ -239,11 +276,16 @@ hash_content_directory() { compute_content_hash() { local path="" + local physical_path="" for path in "$@"; do - if [[ -L "${path}" || -f "${path}" ]]; then + physical_path="${path}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${path}" != /* ]]; then + physical_path="${ROCM_BUILD_CONTEXT_ROOT}/${path}" + fi + if [[ -L "${physical_path}" || -f "${physical_path}" ]]; then hash_content_file "${path}" || return $? - elif [[ -d "${path}" ]]; then + elif [[ -d "${physical_path}" ]]; then hash_content_directory "${path}" || return $? else printf 'missing:%s\n' "${path}" @@ -251,25 +293,69 @@ compute_content_hash() { done | sha256sum | cut -d' ' -f1 } -normalize_ci_worktree_modes() { +validate_ci_build_context_source() { + local source_root="$1" + local context_commit="" + local source_head="" local entry="" local metadata="" local mode="" local stage="" local path="" local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes" - local -a regular_files=() - local -a executable_files=() + local info_attributes="" - [[ "${BUILDKITE:-false}" == "true" ]] || return 0 - [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0 - if ! git ls-files --stage -z > "${index_modes_file}"; then - echo "Failed to read Git modes for the CI Docker context" >&2 + if ! context_commit=$(git -C "${source_root}" rev-parse \ + --verify "${BUILDKITE_COMMIT:-HEAD}^{commit}"); then + echo "Failed to resolve the CI Docker context revision" >&2 + return 1 + fi + if ! source_head=$(git -C "${source_root}" rev-parse --verify HEAD); then + echo "Failed to resolve the checked-out CI revision" >&2 + return 1 + fi + if [[ "${context_commit}" != "${source_head}" ]]; then + echo "BUILDKITE_COMMIT does not match the checked-out CI revision" >&2 + return 1 + fi + ROCM_BUILD_CONTEXT_COMMIT="${context_commit}" + if ! git -C "${source_root}" diff \ + --quiet --no-ext-diff --ignore-submodules=none --; then + echo "Tracked worktree changes cannot be omitted from the CI Docker context" \ + >&2 + return 1 + fi + if ! git -C "${source_root}" diff --cached \ + --quiet --no-ext-diff --ignore-submodules=none HEAD --; then + echo "Staged changes cannot be omitted from the CI Docker context" >&2 + return 1 + fi + # Untracked and ignored worker outputs are intentionally absent: the + # pinned Git tree, rather than mutable checkout contents, is the contract. + if ! info_attributes=$(git -C "${source_root}" \ + rev-parse --path-format=absolute --git-path info/attributes); then + echo "Failed to locate repository-local Git attributes" >&2 + return 1 + fi + if [[ -s "${info_attributes}" ]]; then + echo "Repository-local Git attributes cannot define the CI Docker context" \ + >&2 return 1 fi - # Some shared workspaces expose 0644/0755 files as 0664/0775. Restore the - # index modes so both the identity hash and Docker context are canonical. + ROCM_BUILD_CONTEXT_INDEX="${SCRIPT_TMP_DIR}/docker-context.index" + if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \ + git -C "${source_root}" -c core.splitIndex=false \ + read-tree "${context_commit}^{tree}"; then + echo "Failed to create the CI Docker context index" >&2 + return 1 + fi + if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \ + git -C "${source_root}" ls-files --stage -z \ + > "${index_modes_file}"; then + echo "Failed to read the Git index for the CI Docker context" >&2 + return 1 + fi while IFS= read -r -d '' entry; do metadata="${entry%%$'\t'*}" mode="${metadata%% *}" @@ -280,13 +366,12 @@ normalize_ci_worktree_modes() { return 1 fi case "${mode}" in - 100644) - regular_files+=("${path}") + 100644|100755|120000) ;; - 100755) - executable_files+=("${path}") - ;; - 120000|160000) + 160000) + echo "Git submodule cannot be materialized in the CI Docker context: ${path}" \ + >&2 + return 1 ;; *) echo "Unsupported Git mode ${mode} for ${path}" >&2 @@ -294,10 +379,130 @@ normalize_ci_worktree_modes() { ;; esac done < "${index_modes_file}" +} - ((${#regular_files[@]} == 0)) || chmod 0644 -- "${regular_files[@]}" - ((${#executable_files[@]} == 0)) || chmod 0755 -- "${executable_files[@]}" - echo "Normalized Git file modes for the CI Docker context" +describe_ci_revision() { + git -C "$1" describe --tags --long --abbrev=10 \ + --match '*[0-9]*' "$2" 2>/dev/null +} + +write_ci_git_archival_metadata() { + local source_root="$1" + local context_root="$2" + local commit="${ROCM_BUILD_CONTEXT_COMMIT}" + local commit_date="" + local describe="" + local is_shallow="false" + + commit_date=$(git -C "${source_root}" show -s --format=%cI "${commit}") \ + || return $? + is_shallow=$(git -C "${source_root}" rev-parse --is-shallow-repository) \ + || return $? + if git -C "${source_root}" remote get-url origin >/dev/null 2>&1; then + # AMD agents use shallow, no-tag clones. Reach a release tag so one + # commit cannot acquire different versions from different depths. + # Versioning only needs tags and commits, not historical source trees. + echo "Synchronizing version tags for the canonical CI Docker context" + (cd "${source_root}" && git_fetch_with_timeout --quiet \ + --filter=tree:0 --prune origin '+refs/tags/*:refs/tags/*') \ + || return $? + if [[ "${is_shallow}" == "true" ]] \ + && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then + echo "Deepening history to reach a version tag" + (cd "${source_root}" && git_fetch_with_timeout --quiet \ + --filter=tree:0 --no-tags --deepen=1000 origin "${commit}") \ + || return $? + is_shallow=$(git -C "${source_root}" \ + rev-parse --is-shallow-repository) || return $? + if [[ "${is_shallow}" == "true" ]] \ + && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then + echo "No version tag within 1,000 commits; fetching full history" + (cd "${source_root}" && git_fetch_with_timeout --quiet \ + --filter=tree:0 --no-tags --unshallow origin "${commit}") \ + || return $? + fi + fi + fi + if ! describe=$(describe_ci_revision "${source_root}" "${commit}"); then + echo "No numeric version tag is reachable from the CI revision" >&2 + return 1 + fi + if [[ -e "${context_root}/.git_archival.txt" \ + || -L "${context_root}/.git_archival.txt" ]]; then + echo "The source tree already contains .git_archival.txt" >&2 + return 1 + fi + printf 'node: %s\nnode-date: %s\ndescribe-name: %s\n' \ + "${commit}" "${commit_date}" "${describe}" \ + > "${context_root}/.git_archival.txt" \ + && chmod 0644 -- "${context_root}/.git_archival.txt" +} + +write_build_context_override() { + local escaped_context="" + + [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]] || return 0 + escaped_context=$(hcl_escape_string "${ROCM_BUILD_CONTEXT_ROOT}") || return $? + if ! { + printf 'target "_common-rocm" {\n' + printf ' context = "%s"\n' "${escaped_context}" + printf '}\n' + } > "${BUILD_CONTEXT_OVERRIDE_PATH}"; then + echo "Failed to write the CI Docker context override" >&2 + return 1 + fi + BAKE_FILES+=(-f "${BUILD_CONTEXT_OVERRIDE_PATH}") +} + +prepare_ci_build_context() { + local source_root="" + local context_root="" + + [[ "${BUILDKITE:-false}" == "true" ]] || return 0 + [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0 + + # BuildKit includes file modes in cache keys. Export the pinned revision to + # an owned context instead of changing modes in the shared checkout. + if ! source_root=$(git rev-parse --show-toplevel); then + echo "Failed to locate the CI source checkout" >&2 + return 1 + fi + validate_ci_build_context_source "${source_root}" || return $? + context_root="${SCRIPT_TMP_DIR}/docker-context" + if ! mkdir -m 0700 -- "${context_root}"; then + echo "Failed to create the owned CI Docker context" >&2 + return 1 + fi + if ! ( + umask 0022 + unset GIT_LFS_SKIP_SMUDGE + export GIT_ATTR_NOSYSTEM=1 + export GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" + git -C "${source_root}" \ + -c core.attributesFile=/dev/null \ + -c core.autocrlf=false \ + -c core.eol=lf \ + -c core.symlinks=true \ + --work-tree="${context_root}" \ + checkout-index --all + ); then + echo "Failed to materialize the CI Docker context from the Git index" >&2 + return 1 + fi + + # setuptools-scm understands Git's stable archive format, so full-source + # wheels retain their exact version without copying mutable Git history. + if ! is_ci_base_target; then + write_ci_git_archival_metadata "${source_root}" "${context_root}" \ + || return $? + fi + if [[ -e "${context_root}/.git" ]]; then + echo "Canonical CI Docker context unexpectedly contains .git" >&2 + return 1 + fi + ROCM_BUILD_CONTEXT_ROOT="${context_root}" + BAKE_ALLOW_ARGS+=(--allow "fs.read=${ROCM_BUILD_CONTEXT_ROOT}") + echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}" } compose_dependency_cache_key() { @@ -312,6 +517,12 @@ compose_dependency_cache_key() { hash_dockerfile_stages() { local dockerfile="$1" local stages="$2" + local physical_dockerfile="" + + physical_dockerfile="${dockerfile}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}" + fi awk -v wanted_stages="${stages}" ' BEGIN { @@ -335,14 +546,19 @@ hash_dockerfile_stages() { emit { print } - ' "${dockerfile}" + ' "${physical_dockerfile}" } discover_dockerfile_stage_args() { local dockerfile="$1" local stages="$2" + local physical_dockerfile="" - [[ -f "${dockerfile}" ]] || return 0 + physical_dockerfile="${dockerfile}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}" + fi + [[ -f "${physical_dockerfile}" ]] || return 0 awk -v wanted_stages="${stages}" ' function add_arg(name) { @@ -406,7 +622,7 @@ discover_dockerfile_stage_args() { } } } - ' "${dockerfile}" + ' "${physical_dockerfile}" } get_content_arg_names() { @@ -446,7 +662,7 @@ compute_ci_base_content_hash() { || return 1 if [[ -n "${stages}" ]]; then printf 'dockerfile-stages:%s\n' "${stages}" - if [[ -f "${dockerfile}" ]]; then + if content_regular_file "${dockerfile}"; then hash_dockerfile_stages "${dockerfile}" "${stages}" else printf 'missing:%s\n' "${dockerfile}" @@ -459,9 +675,14 @@ compute_ci_base_content_hash() { extract_dockerfile_arg_default() { local dockerfile="$1" local arg_name="$2" + local physical_dockerfile="" + physical_dockerfile="${dockerfile}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}" + fi sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ - "${dockerfile}" | head -1 + "${physical_dockerfile}" | head -1 } resolve_image_digest() { @@ -523,7 +744,7 @@ resolve_dockerfile_arg_value() { if [[ -z "${value}" && "${env_name}" != "${arg_name}" ]]; then value="${!arg_name:-}" fi - if [[ -z "${value}" && -f "${dockerfile}" ]]; then + if [[ -z "${value}" ]] && content_regular_file "${dockerfile}"; then value=$(extract_dockerfile_arg_default "${dockerfile}" "${arg_name}") fi @@ -741,6 +962,7 @@ init_config() { CI_BASE_LABEL_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/ci-base-label-override.hcl" CSRC_CACHE_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-csrc-cache-override.hcl" ROCM_ARG_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-arg-override.hcl" + BUILD_CONTEXT_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/build-context-override.hcl" BAKE_CONFIG_FILE="bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json" } @@ -1270,7 +1492,8 @@ prepare_git_cache_metadata() { if [[ -z "${PARENT_COMMIT:-}" || -z "${VLLM_MERGE_BASE_COMMIT:-}" ]] \ && git rev-parse --is-shallow-repository 2>/dev/null | grep -q "true"; then echo "Shallow clone detected - deepening for cache key computation" - git_fetch_for_cache --deepen=1 origin || true + git_fetch_for_cache --filter=tree:0 --no-tags --deepen=1 \ + origin "$(git rev-parse HEAD)" || true fi if [[ -z "${PARENT_COMMIT:-}" ]]; then @@ -1524,7 +1747,7 @@ compute_rocm_csrc_content_hash() { printf 'resolved-build-args:\n' hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}" printf 'dockerfile-stages:%s\n' "${stages}" - if [[ -f "${dockerfile_rocm}" ]]; then + if content_regular_file "${dockerfile_rocm}"; then hash_dockerfile_stages "${dockerfile_rocm}" "${stages}" else printf 'missing:%s\n' "${dockerfile_rocm}" @@ -1574,7 +1797,7 @@ compute_rocm_rust_content_hash() { printf 'resolved-build-args:\n' hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}" printf 'dockerfile-stages:%s\n' "${stages}" - if [[ -f "${dockerfile_rocm}" ]]; then + if content_regular_file "${dockerfile_rocm}"; then hash_dockerfile_stages "${dockerfile_rocm}" "${stages}" else printf 'missing:%s\n' "${dockerfile_rocm}" @@ -1736,24 +1959,24 @@ should_export_content_cache_ref() { case "${mode}" in always) - echo "${cache_name} content cache export mode is always; exporting ${cache_ref}" + echo "${cache_name} content cache export mode is always" return 0 ;; never) - echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}" + echo "${cache_name} content cache export mode is never" return 1 ;; missing|"") - if remote_image_exists "${trusted_ref}"; then - echo "${cache_name} trusted content cache exists; not exporting ${cache_ref}" + if registry_ref_exists_with_retry imagetools "${trusted_ref}"; then + echo "${cache_name} trusted content cache is visible: ${trusted_ref}" return 1 fi if [[ "${cache_ref}" != "${trusted_ref}" ]] \ - && remote_image_exists "${cache_ref}"; then - echo "${cache_name} scoped content cache exists; not re-exporting ${cache_ref}" + && registry_ref_exists_with_retry imagetools "${cache_ref}"; then + echo "${cache_name} scoped content cache is visible: ${cache_ref}" return 1 fi - echo "${cache_name} content cache missing; will export ${cache_ref}" + echo "${cache_name} content cache is missing: ${cache_ref}" return 0 ;; *) @@ -1791,7 +2014,7 @@ write_rocm_cache_override() { validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE" echo "ROCm content cache export mode: ${content_cache_export_mode}" echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}" - echo "ROCm Rust cache export mode: ${rust_cache_to_mode}" + echo "ROCm Rust fallback cache export mode: ${rust_cache_to_mode}" echo "ROCm final image cache export mode: ${rocm_cache_to_mode}" if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then @@ -1825,15 +2048,21 @@ write_rocm_cache_override() { "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}" ) fi - if should_export_content_cache_ref \ + # Legacy commit/branch exports are only needed while the exact-input + # content ref is absent. The exact ref itself is refreshed below. + if ! should_export_content_cache_ref \ "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust" \ "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"; then - rust_cache_to+=( - "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true" - ) - else export_rust_cache=0 fi + if [[ "${content_cache_export_mode}" != "never" ]]; then + # Refresh the exact-input ref in the original solve regardless of + # which local or remote cache supplied the Rust result. + rust_cache_to+=( + "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=min,ignore-error=true" + ) + echo "ROCm Rust exact-input cache will be refreshed (mode=min): ${ROCM_RUST_CONTENT_CACHE_REF}" + fi fi combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}") @@ -1872,6 +2101,16 @@ write_rocm_cache_override() { ) fi + # Standalone image/wheel targets reach rust-build but not the cache-only + # exporter unless it is requested explicitly. + if ((${#rust_cache_to[@]} > 0)); then + case "${TARGET}" in + test-rocm-ci|export-wheel-rocm) + BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}") + ;; + esac + fi + if [[ "${TARGET}" == "test-rocm-ci-with-wheel" ]]; then export_wheel_cache_to=() else @@ -1937,12 +2176,17 @@ EOF extract_dependency_pins() { local bake_dir="" local dockerfile_rocm="" + local physical_dockerfile="" local var="" local val="" bake_dir=$(dirname "${VLLM_BAKE_FILE}") dockerfile_rocm="${bake_dir}/Dockerfile.rocm" - if [[ ! -f "${dockerfile_rocm}" ]]; then + physical_dockerfile="${dockerfile_rocm}" + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}" + fi + if [[ ! -f "${physical_dockerfile}" ]]; then return 0 fi @@ -1954,7 +2198,7 @@ extract_dependency_pins() { val=$( sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ - "${dockerfile_rocm}" | head -1 + "${physical_dockerfile}" | head -1 ) if [[ -n "${val}" ]]; then export "${var}=${val}" @@ -2162,7 +2406,8 @@ print_bake_config() { echo "--- :page_facing_up: Resolved bake configuration" mapfile -t print_targets < <(bake_config_targets) - docker buildx bake "${BAKE_FILES[@]}" --print "${print_targets[@]}" | tee "${BAKE_CONFIG_FILE}" + docker buildx bake "${BAKE_ALLOW_ARGS[@]}" \ + "${BAKE_FILES[@]}" --print "${print_targets[@]}" | tee "${BAKE_CONFIG_FILE}" if command -v buildkite-agent >/dev/null 2>&1 && [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then buildkite-agent artifact upload "${BAKE_CONFIG_FILE}" || true @@ -2234,6 +2479,7 @@ seed_dependency_caches_if_needed() { echo "--- :docker: Seeding ${target}" echo "Expected cache ref: ${cache_ref}" docker buildx bake \ + "${BAKE_ALLOW_ARGS[@]}" \ "${BAKE_FILES[@]}" \ --progress "${BUILDKIT_PROGRESS:-plain}" \ "${target}" @@ -2250,6 +2496,7 @@ run_bake() { echo "--- :docker: Building ${TARGET}" docker buildx bake \ + "${BAKE_ALLOW_ARGS[@]}" \ "${BAKE_FILES[@]}" \ --progress "${BUILDKIT_PROGRESS:-plain}" \ "${BAKE_TARGETS[@]}" @@ -2358,14 +2605,21 @@ main() { configure_ci_base_write_scope print_header validate_inputs - normalize_ci_worktree_modes load_ci_hcl init_bake_files + if is_ci_base_target; then + prepare_ci_build_context + fi compute_ci_base_hash_if_needed configure_ci_base_image_refs maybe_skip_existing_image setup_builder prepare_git_cache_metadata + # Non-ci_base builds may deepen a shallow checkout above. Derive archival + # version metadata only after that lookup sees the available tag history. + if ! is_ci_base_target; then + prepare_ci_build_context + fi extract_dependency_pins write_rocm_build_arg_override compute_dependency_cache_keys @@ -2373,6 +2627,8 @@ main() { compute_rocm_csrc_content_hash_if_needed compute_rocm_rust_content_hash_if_needed write_rocm_cache_override + # Keep the context override last so every bake target uses the owned tree. + write_build_context_override resolve_ci_base_dependency_targets print_bake_config if [[ "${BAKE_PRINT_ONLY:-0}" == "1" ]]; then diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index f612a0bd0e..4595daa56f 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -5,6 +5,7 @@ ARG VLLM_BRANCH="main" ARG COMMON_WORKDIR=/app ARG BASE_IMAGE=rocm/vllm-dev:base ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base +ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7 # NIC backend for MoRI RDMA support. # By default (all), drivers and userspace libraries for all supported NIC types # (ainic and bnxt) are installed; MoRI selects the appropriate one at runtime. @@ -272,6 +273,20 @@ COPY --from=fetch_vllm \ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ uv pip install --system -r requirements/rocm.txt +# Fetch only the pinned ROCm Triton kernels package. Keeping this before the +# vLLM source COPYs makes the download reusable when native sources change. +FROM base AS rocm-triton-kernels +ARG ROCM_TRITON_KERNELS_COMMIT +ADD https://codeload.github.com/ROCm/triton/tar.gz/${ROCM_TRITON_KERNELS_COMMIT} \ + /tmp/rocm-triton.tar.gz +RUN mkdir -p /opt/rocm-triton-kernels \ + && tar --extract --gzip --file /tmp/rocm-triton.tar.gz \ + --directory /opt/rocm-triton-kernels --strip-components=4 \ + --no-same-owner \ + "triton-${ROCM_TRITON_KERNELS_COMMIT}/python/triton_kernels/triton_kernels" \ + && test -f /opt/rocm-triton-kernels/__init__.py \ + && rm /tmp/rocm-triton.tar.gz + # ----------------------- # vLLM native build stages # @@ -279,6 +294,11 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ # compilation. That keeps unrelated CI/test/docs edits from invalidating the # expensive HIP/C++ build layer. FROM build_vllm_dependencies AS csrc-build +ARG ROCM_TRITON_KERNELS_COMMIT +COPY --from=rocm-triton-kernels \ + /opt/rocm-triton-kernels /opt/rocm-triton-kernels +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels + # pyproject.toml is bind-mounted so it is not persisted in the csrc-build image, # but it still participates in the BuildKit cache key for this RUN step. COPY setup.py CMakeLists.txt ./ @@ -292,13 +312,15 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build" RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \ --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ - export CCACHE_BASEDIR="$PWD" \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + cmake/external_projects/triton_kernels.cmake \ + && export CCACHE_BASEDIR="$PWD" \ && echo "=== ccache stats before ROCm native build ===" \ && (ccache --show-stats || true) \ && (ccache --zero-stats || true) \ && EFFECTIVE_MAX_JOBS="${MAX_JOBS:-$(nproc)}" \ && echo "Building ROCm native extension wheel with MAX_JOBS=${EFFECTIVE_MAX_JOBS}" \ - && LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \ + && PYTHONDONTWRITEBYTECODE=1 LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \ && test -d dist \ && ls dist/*.whl >/dev/null \ && echo "=== ccache stats after ROCm native build ===" \ @@ -322,7 +344,7 @@ RUN cd "${COMMON_WORKDIR}/vllm" \ && export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \ && export VLLM_DOCKER_BUILD_CONTEXT=1 \ && echo "Packaging vLLM ROCm wheel using precompiled extensions from ${VLLM_PRECOMPILED_WHEEL_LOCATION}" \ - && python3 setup.py bdist_wheel --dist-dir=dist \ + && PYTHONDONTWRITEBYTECODE=1 python3 setup.py bdist_wheel --dist-dir=dist \ && test -d dist \ && ls dist/*.whl >/dev/null FROM scratch AS export_vllm @@ -551,6 +573,10 @@ RUN /bin/bash -lc 'set -euo pipefail; \ FROM fetch_vllm AS build_vllm_wheel_release ARG COMMON_WORKDIR +ARG ROCM_TRITON_KERNELS_COMMIT +COPY --from=rocm-triton-kernels \ + /opt/rocm-triton-kernels /opt/rocm-triton-kernels +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels # Drop the pre-built Rust artifacts into the source tree. setup.py detects # them and ships them as-is, skipping the local Rust build. @@ -590,7 +616,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \ # This ensures setuptools_scm sees clean repo state for version detection RUN --mount=type=bind,source=.git,target=vllm/.git \ --mount=type=cache,target=/root/.cache/uv \ - cd vllm \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + vllm/cmake/external_projects/triton_kernels.cmake \ + && cd vllm \ && uv pip install --system setuptools_scm regex \ && VLLM_VERSION=$(python3 -c "import setuptools_scm; print(setuptools_scm.get_version())") \ && echo "Detected vLLM version: ${VLLM_VERSION}" \