From 7f6432cc0daf97c5b9a2df141641c595a766ff40 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 9 Aug 2026 03:25:03 -0500 Subject: [PATCH] [ROCm][CI] Reuse equivalent ROCm CI images (#48646) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/ci_config_rocm.yaml | 13 + .buildkite/hardware_tests/amd.yaml | 20 +- .buildkite/scripts/ci-bake-rocm.sh | 1194 ++++++++++------- .../scripts/hardware_ci/run-amd-test.sh | 247 ++++ .buildkite/scripts/rocm/build-ci-base.sh | 22 +- .buildkite/scripts/rocm/build-test-image.sh | 73 +- .buildkite/scripts/rocm/refresh-base-image.sh | 749 ++++++----- .buildkite/scripts/rocm/smoke-test-image.sh | 13 +- .dockerignore | 3 + docker/Dockerfile.rocm | 43 +- docker/Dockerfile.rocm_base | 63 +- docker/ci-rocm.hcl | 39 +- .../tools/test_docker_build_metadata_args.py | 4 - 13 files changed, 1553 insertions(+), 930 deletions(-) diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml index 408ccee58e..cc7c2ad27f 100644 --- a/.buildkite/ci_config_rocm.yaml +++ b/.buildkite/ci_config_rocm.yaml @@ -2,11 +2,13 @@ name: vllm_rocm_ci job_dirs: - ".buildkite/hardware_tests" run_all_patterns: + - ".dockerignore" - "docker/Dockerfile.rocm" - "docker/Dockerfile.rocm_base" - "docker/ci-rocm.hcl" - "docker/docker-bake-rocm.hcl" - ".buildkite/hardware_tests/amd.yaml" + - ".buildkite/ci_config_rocm.yaml" - ".buildkite/scripts/ci-bake-rocm.sh" - ".buildkite/scripts/rocm/" - ".buildkite/scripts/hardware_ci/run-amd-test.py" @@ -14,9 +16,20 @@ run_all_patterns: - "CMakeLists.txt" - "requirements/common.txt" - "requirements/rocm.txt" + - "requirements/build/rust.txt" - "requirements/build/rocm.txt" - "requirements/test/rocm.txt" + - "pyproject.toml" - "setup.py" + - "build_rust.sh" + - "rust/" + - "rust-toolchain.toml" + - "tools/build_rust.py" + - "tools/install_protoc.sh" + - "tools/install_torchcodec_rocm.sh" + - "tests/vllm_test_utils/" + - "vllm/envs.py" + - "vllm/__init__.py" - "csrc/" - "cmake/" run_all_exclude_patterns: diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml index bd73c8102c..66d8a589ab 100644 --- a/.buildkite/hardware_tests/amd.yaml +++ b/.buildkite/hardware_tests/amd.yaml @@ -1,9 +1,9 @@ group: Hardware - AMD Build # ROCm image flow: -# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes. -# 2. Build ci_base from either the stable base or the freshly refreshed base. -# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run. +# 1. Select the content-matching ROCm base, building only on a cache miss. +# 2. Build ci_base from either the stable base or the newly selected base. +# 3. Build the build-scoped ROCm CI image and smoke-test it before GPU jobs run. steps: - label: "AMD: :docker: refresh ROCm base" key: refresh-rocm-base-amd @@ -12,6 +12,8 @@ steps: no_plugin: true commands: - bash .buildkite/scripts/rocm/refresh-base-image.sh + concurrency: 1 + concurrency_group: "vllm/rocm/base-build/$BUILDKITE_COMMIT" env: DOCKER_BUILDKIT: "1" BUILDKIT_PROGRESS: "tty" @@ -38,18 +40,21 @@ steps: no_plugin: true commands: - bash .buildkite/scripts/rocm/build-ci-base.sh + concurrency: 1 + concurrency_group: "vllm/rocm/ci-base-build/$BUILDKITE_COMMIT" env: DOCKER_BUILDKIT: "1" BUILDKIT_PROGRESS: "tty" TERM: "xterm-256color" VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" - REMOTE_VLLM: "1" - VLLM_BRANCH: "$BUILDKITE_COMMIT" + REMOTE_VLLM: "0" retry: automatic: - exit_status: 1 # Transient Docker/BuildKit failure limit: 1 + - exit_status: 2 # Exact image found but registry handoff failed + limit: 1 - exit_status: -1 # Agent was lost limit: 1 - exit_status: -10 # Agent was lost @@ -71,9 +76,8 @@ steps: TERM: "xterm-256color" VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" - IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT" - REMOTE_VLLM: "1" - VLLM_BRANCH: "$BUILDKITE_COMMIT" + IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" + REMOTE_VLLM: "0" retry: automatic: - exit_status: 1 # Transient Docker/BuildKit failure diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 31d56bdc29..b0f32cd56f 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -3,7 +3,7 @@ # # The wrapper keeps three build concerns separate: # * ci_base builds are content-addressed by vllm.ci_base.content_hash. -# * test image builds are commit-addressed by org.opencontainers.image.revision. +# * test image contents record the commit while runtime tags stay build-scoped. # * ROCm install artifacts are uploaded only for targets that export wheels. # # Usage: @@ -15,11 +15,20 @@ set -euo pipefail DEFAULT_REPO_SLUG="vllm-project/vllm" DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl" -DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh" +DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils" DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm" -DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_nixl build_rocshmem build_deepep mori_base ci_base" -DEFAULT_CI_BASE_METADATA_VERSION="1" -IMAGE_EXISTED_BEFORE_BUILD=0 +DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl build_rocshmem build_deepep mori_base ci_base" +DEFAULT_CI_BASE_METADATA_VERSION="3" +# ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected +# local-source stages rather than unreachable remote-fetch alternatives. +DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py" +DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies csrc-build" +DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/install_protoc.sh build_rust.sh" +DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build" +# Docker's 128-character tag limit minus the longest cache prefix +# ("csrc-rocm-branch-" and "rust-rocm-branch-", both 17 characters). +ROCM_CACHE_BRANCH_TAG_MAX_LEN=111 +CI_BASE_WRITE_SCOPE="" TARGET="" CI_HCL_SOURCE="${CI_HCL_SOURCE:-}" @@ -86,10 +95,11 @@ compose_cache_branch_tag() { local suffix="" local prefix="" local max_prefix_len=0 + local max_tag_len="${ROCM_CACHE_BRANCH_TAG_MAX_LEN}" suffix="$(cache_scope_suffix)" prefix="$(clean_docker_tag "${repo_slug}")-$(clean_docker_tag "${branch}")" - max_prefix_len=$((128 - ${#suffix} - 1)) + max_prefix_len=$((max_tag_len - ${#suffix} - 1)) if (( max_prefix_len < 1 )); then max_prefix_len=1 fi @@ -112,6 +122,57 @@ parse_repo_slug() { printf '%s\n' "${repo_slug}" } +normalize_repo_slug() { + local repo_slug="${1:-}" + + repo_slug="${repo_slug%/}" + repo_slug="${repo_slug%.git}" + repo_slug="${repo_slug#https://github.com/}" + repo_slug="${repo_slug#http://github.com/}" + repo_slug="${repo_slug#ssh://git@github.com/}" + repo_slug="${repo_slug#git@github.com:}" + repo_slug="${repo_slug#github.com/}" + printf '%s\n' "${repo_slug}" +} + +is_trusted_ci_cache_writer() { + local actual_repo="" + local trusted_repo="" + + [[ "${BUILDKITE:-false}" == "true" ]] || return 1 + [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1 + [[ "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]] || return 1 + actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}") + trusted_repo=$(normalize_repo_slug \ + "${CI_BASE_STABLE_REPO_SLUG:-${DEFAULT_REPO_SLUG}}") + [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]] +} + +ci_base_write_scope() { + local identity="" + local source_repo="${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-local}}" + + if is_trusted_ci_cache_writer; then + return 0 + fi + identity=$(printf '%s\n' "${source_repo}" | sha256sum | cut -c1-12) + printf 'preview-%s\n' "${identity}" +} + +configure_ci_base_write_scope() { + local scope="" + + scope=$(ci_base_write_scope) + if [[ -n "${scope}" ]]; then + CI_BASE_WRITE_SCOPE=$(clean_docker_tag "${scope}") + echo "Non-canonical cache writes use source scope: ${CI_BASE_WRITE_SCOPE}" + else + CI_BASE_WRITE_SCOPE="" + echo "Trusted main build: publishing canonical ci_base refs" + fi + export CI_BASE_WRITE_SCOPE +} + get_buildkite_repo_slug() { parse_repo_slug "${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-}}" } @@ -145,25 +206,100 @@ hash_string_short() { printf '%s' "$1" | sha256sum | cut -c1-16 } +list_content_files() { + # Hash the checkout inputs Docker can intentionally consume, not ignored + # compiler/test debris left behind on a reused worker. + git ls-files -z --cached --others --exclude-standard -- "$1" \ + | LC_ALL=C sort -z +} + +hash_content_file() { + local file="$1" + + if [[ -L "${file}" ]]; then + printf 'symlink:%s\ntarget:%s\n' "${file}" "$(readlink "${file}")" + return + fi + if [[ ! -f "${file}" ]]; then + printf 'missing:%s\n' "${file}" + return + fi + printf 'file:%s\nmode:%s\n' "${file}" "$(stat -c '%a' "${file}")" + sha256sum "${file}" +} + +hash_content_directory() { + if ! list_content_files "$1" | while IFS= read -r -d '' file; do + hash_content_file "${file}" || exit $? + done; then + echo "Failed to hash content under $1" >&2 + return 1 + fi +} + compute_content_hash() { - local path - local file + local path="" for path in "$@"; do - if [[ -d "${path}" ]]; then - while IFS= read -r -d '' file; do - printf 'file:%s\n' "${file}" - sha256sum "${file}" - done < <(find "${path}" -type f -print0 | sort -z) - elif [[ -f "${path}" ]]; then - printf 'file:%s\n' "${path}" - sha256sum "${path}" + if [[ -L "${path}" || -f "${path}" ]]; then + hash_content_file "${path}" || return $? + elif [[ -d "${path}" ]]; then + hash_content_directory "${path}" || return $? else printf 'missing:%s\n' "${path}" fi done | sha256sum | cut -d' ' -f1 } +normalize_ci_worktree_modes() { + local entry="" + local metadata="" + local mode="" + local stage="" + local path="" + local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes" + local -a regular_files=() + local -a executable_files=() + + [[ "${BUILDKITE:-false}" == "true" ]] || return 0 + [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0 + if ! git ls-files --stage -z > "${index_modes_file}"; then + echo "Failed to read Git modes for the CI Docker context" >&2 + return 1 + fi + + # Some shared workspaces expose 0644/0755 files as 0664/0775. Restore the + # index modes so both the identity hash and Docker context are canonical. + while IFS= read -r -d '' entry; do + metadata="${entry%%$'\t'*}" + mode="${metadata%% *}" + stage="${metadata##* }" + path="${entry#*$'\t'}" + if [[ "${stage}" != "0" ]]; then + echo "Unmerged Git entry cannot be used as Docker context: ${path}" >&2 + return 1 + fi + case "${mode}" in + 100644) + regular_files+=("${path}") + ;; + 100755) + executable_files+=("${path}") + ;; + 120000|160000) + ;; + *) + echo "Unsupported Git mode ${mode} for ${path}" >&2 + return 1 + ;; + esac + done < "${index_modes_file}" + + ((${#regular_files[@]} == 0)) || chmod 0644 -- "${regular_files[@]}" + ((${#executable_files[@]} == 0)) || chmod 0755 -- "${executable_files[@]}" + echo "Normalized Git file modes for the CI Docker context" +} + compose_dependency_cache_key() { local prefix="$1" local material="$2" @@ -185,7 +321,7 @@ hash_dockerfile_stages() { wanted[stage_list[idx]] = 1 } } - emit = 1 + emit = 0 } $1 == "FROM" { stage = "" @@ -285,9 +421,10 @@ get_content_arg_names() { fi | awk 'NF && !seen[$0]++' } -compute_ci_base_content_hash_once() { +compute_ci_base_content_hash() { local -a content_paths=() local -a content_args=() + local content_files_hash="" local dockerfile="${CI_BASE_DOCKERFILE:-}" local stages="${CI_BASE_DOCKERFILE_STAGES:-}" @@ -295,9 +432,13 @@ compute_ci_base_content_hash_once() { mapfile -t content_args < <( get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}" ) + if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then + echo "Failed to hash ci_base content files" >&2 + return 1 + fi { - printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")" + printf 'content-files-hash:%s\n' "${content_files_hash}" if [[ -n "${dockerfile}" ]]; then printf 'dockerfile:%s\n' "${dockerfile}" printf 'resolved-build-args:\n' @@ -315,53 +456,6 @@ compute_ci_base_content_hash_once() { } | sha256sum | cut -d' ' -f1 } -compute_ci_base_content_hash() { - local attempts="${CI_BASE_HASH_ATTEMPTS:-3}" - local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}" - local attempt=0 - local hash="" - local failed=0 - local -a hashes=() - - if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then - echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2 - return 1 - fi - if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then - echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2 - return 1 - fi - - for ((attempt = 1; attempt <= attempts; attempt++)); do - if ! hash=$(compute_ci_base_content_hash_once); then - echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2 - failed=1 - else - hashes+=("${hash}") - echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2 - fi - - if ((attempt < attempts)); then - sleep "${delay_secs}" - fi - done - - if ((failed)) || ((${#hashes[@]} != attempts)); then - echo "Could not calculate a reliable ci_base content hash" >&2 - return 1 - fi - - for hash in "${hashes[@]:1}"; do - if [[ "${hash}" != "${hashes[0]}" ]]; then - echo "ci_base content hash changed between calculations" >&2 - printf ' observed: %s\n' "${hashes[@]}" >&2 - return 1 - fi - done - - printf '%s\n' "${hashes[0]}" -} - extract_dockerfile_arg_default() { local dockerfile="$1" local arg_name="$2" @@ -372,10 +466,42 @@ extract_dockerfile_arg_default() { resolve_image_digest() { local image_ref="$1" + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}" + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}" + local attempt=0 + local digest="" + local output="" + local status=0 - docker buildx imagetools inspect "${image_ref}" 2>/dev/null \ - | sed -n -E 's/^Digest:[[:space:]]+//p' \ - | head -1 + if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" + return + fi + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid image digest retry configuration" >&2 + return 1 + fi + + for ((attempt = 1; attempt <= attempts; attempt++)); do + status=0 + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$? + digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}") + if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + printf '%s\n' "${digest}" + return + fi + if ((attempt < attempts)); then + printf \ + 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \ + "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2 + sleep "${delay_secs}" + fi + done + + printf 'Failed to resolve digest for %s (status %d)\n%s\n' \ + "${image_ref}" "${status}" "${output:-}" >&2 + return 1 } resolve_dockerfile_arg_value() { @@ -388,6 +514,9 @@ resolve_dockerfile_arg_value() { ARG_PYTORCH_ROCM_ARCH) env_name="PYTORCH_ROCM_ARCH" ;; + max_jobs) + env_name="CI_MAX_JOBS" + ;; esac value="${!env_name:-}" @@ -411,18 +540,35 @@ hash_dockerfile_arg_values() { for arg_name in "$@"; do [[ -n "${arg_name}" ]] || continue arg_value=$(resolve_dockerfile_arg_value "${dockerfile}" "${arg_name}") - printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-}" if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then - digest=$(resolve_image_digest "${arg_value}") - if [[ -z "${digest}" ]]; then + if ! digest=$(resolve_image_digest "${arg_value}"); then echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2 return 1 fi printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}" + else + printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-}" fi done } +pin_base_image() { + local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}" + local base_image="" + local digest="" + + base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE") + [[ -n "${base_image}" ]] || return 0 + if ! digest=$(resolve_image_digest "${base_image}"); then + echo "Error: could not resolve base image digest for ${base_image}" >&2 + echo "Refusing to compute content-addressed cache keys from a mutable tag." >&2 + return 1 + fi + BASE_IMAGE="${base_image%@*}@${digest}" + export BASE_IMAGE + echo "Pinned base image for this build: ${BASE_IMAGE}" +} + is_ci_base_target() { [[ "${TARGET}" == *"ci-base-rocm"* ]] } @@ -433,11 +579,6 @@ is_commit_image_target() { return 0 } -image_tag_is_commit_scoped() { - [[ -n "${IMAGE_TAG:-}" && -n "${BUILDKITE_COMMIT:-}" ]] || return 1 - [[ "${IMAGE_TAG}" == *"${BUILDKITE_COMMIT}"* ]] -} - should_upload_wheel_artifacts() { [[ "${UPLOAD_ROCM_WHEEL_ARTIFACTS:-0}" == "1" ]] && return 0 [[ "${TARGET}" == *"with-wheel"* \ @@ -445,168 +586,99 @@ should_upload_wheel_artifacts() { || "${TARGET}" == *"artifact"* ]] } -set_buildkite_metadata() { - local key="$1" - local value="$2" - - [[ -n "${value}" ]] || return 0 - if command -v buildkite-agent >/dev/null 2>&1; then - buildkite-agent meta-data set "${key}" "${value}" || true - fi -} - get_remote_image_label() { local image_ref="$1" local label_key="$2" + local format="{{ index .Image.Config.Labels \"${label_key}\" }}" - docker buildx imagetools inspect "${image_ref}" --raw 2>/dev/null \ - | python3 -c ' -import json -import subprocess -import sys -import urllib.parse -import urllib.request - -image_ref = sys.argv[1] -label_key = sys.argv[2] - - -def docker_hub_repo(image_name): - image_name = image_name.split("@", 1)[0] - last_component = image_name.rsplit("/", 1)[-1] - if ":" in last_component: - image_name = image_name.rsplit(":", 1)[0] - - parts = image_name.split("/") - if len(parts) > 1 and ( - "." in parts[0] or ":" in parts[0] or parts[0] == "localhost" - ): - registry = parts[0] - if registry not in { - "docker.io", - "index.docker.io", - "registry-1.docker.io", - }: - return None - image_name = "/".join(parts[1:]) - elif len(parts) == 1: - image_name = f"library/{image_name}" - - return image_name - - -try: - data = json.load(sys.stdin) - if data.get("manifests"): - manifest = next( - ( - entry - for entry in data["manifests"] - if entry.get("platform", {}).get("os") != "unknown" - and entry.get("platform", {}).get("architecture") != "unknown" - ), - data["manifests"][0], - ) - digest = manifest["digest"] - result = subprocess.run( - [ - "docker", - "buildx", - "imagetools", - "inspect", - image_ref + "@" + digest, - "--raw", - ], - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0 or not result.stdout: - raise RuntimeError("digest inspect failed") - data = json.loads(result.stdout) - - annotations = data.get("annotations", {}) - if label_key in annotations: - print(annotations[label_key]) - raise SystemExit(0) - - config_digest = data.get("config", {}).get("digest") - if not config_digest: - print("") - raise SystemExit(0) - - image_name = docker_hub_repo(image_ref) - if not image_name: - print("") - raise SystemExit(0) - - token_url = ( - "https://auth.docker.io/token?" - + urllib.parse.urlencode( - { - "service": "registry.docker.io", - "scope": f"repository:{image_name}:pull", - } - ) - ) - with urllib.request.urlopen(token_url, timeout=30) as response: - token = json.load(response)["token"] - - request = urllib.request.Request( - f"https://registry-1.docker.io/v2/{image_name}/blobs/{config_digest}", - headers={"Authorization": f"Bearer {token}"}, - ) - with urllib.request.urlopen(request, timeout=30) as response: - config_blob = json.load(response) - - labels = config_blob.get("config", {}).get("Labels", {}) - print(labels.get(label_key, "")) -except Exception: - print("") -' "${image_ref}" "${label_key}" 2>/dev/null || echo "" + docker buildx imagetools inspect "${image_ref}" \ + --format "${format}" 2>/dev/null | awk 'NF { print; exit }' || true } -get_remote_image_label_with_retry() { +remote_ci_base_identity_is_current_with_retry() { local image_ref="$1" - local label_key="$2" - local attempts="${3:-6}" - local delay_secs="${4:-5}" - local label_value="" - local attempt + local attempts="${CI_BASE_LABEL_ATTEMPTS:-4}" + local delay_secs="${CI_BASE_LABEL_RETRY_DELAY:-2}" + local identity="" + local remote_hash="" + local remote_version="" + local content_files_hash="" + local base_digest="" + local attempt=0 + local expected_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" + local format='{{ index .Image.Config.Labels "vllm.ci_base.content_hash" }}|{{ index .Image.Config.Labels "vllm.ci_base.metadata_version" }}|{{ index .Image.Config.Labels "vllm.ci_base.content_files_hash" }}|{{ index .Image.Config.Labels "vllm.rocm.base_image_digest" }}' + + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid ci_base label retry configuration" >&2 + return 1 + fi for ((attempt = 1; attempt <= attempts; attempt++)); do - label_value=$(get_remote_image_label "${image_ref}" "${label_key}") - if [[ -n "${label_value}" ]]; then - printf '%s\n' "${label_value}" + identity=$(docker buildx imagetools inspect "${image_ref}" \ + --format "${format}" 2>/dev/null || true) + IFS='|' read -r \ + remote_hash remote_version content_files_hash base_digest <<< "${identity}" + if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH:-}" \ + && "${remote_version}" == "${expected_version}" \ + && "${content_files_hash}" =~ ^[0-9a-f]{64}$ \ + && "${base_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then return 0 fi - if [[ ${attempt} -lt ${attempts} ]]; then + ((attempt == attempts)) || sleep "${delay_secs}" + done + + echo "ci_base identity did not match after ${attempts} attempts: ${image_ref}" >&2 + echo " expected hash/version: ${CI_BASE_CONTENT_HASH:-}/${expected_version}" >&2 + echo " observed hash/version: ${remote_hash:-}/${remote_version:-}" >&2 + return 1 +} + +registry_ref_exists_with_retry() { + local inspect_kind="$1" + local image_ref="$2" + local attempts="${ROCM_REGISTRY_PROBE_ATTEMPTS:-2}" + local delay_secs="${ROCM_REGISTRY_PROBE_RETRY_DELAY:-1}" + local output="" + local status=0 + local attempt=0 + + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid registry probe retry configuration" >&2 + return 1 + fi + + for ((attempt = 1; attempt <= attempts; attempt++)); do + status=0 + if [[ "${inspect_kind}" == "manifest" ]]; then + output=$(docker manifest inspect "${image_ref}" 2>&1) || status=$? + else + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) \ + || status=$? + fi + if ((status == 0)); then + return 0 + fi + # A definitive registry miss is normal for a new content key. Retry + # transport/rate-limit failures, but do not add latency to a real 404. + if grep -Eiq \ + 'manifest unknown|no such manifest|name unknown|not found|does not exist' \ + <<< "${output}"; then + return 1 + fi + if ((attempt < attempts)); then + echo "Registry probe failed for ${image_ref} (${attempt}/${attempts}); retrying" >&2 sleep "${delay_secs}" fi done - return 0 -} - -remote_ci_base_metadata_is_current() { - local image_ref="$1" - local metadata_version="" - - metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version") - [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]] -} - -remote_ci_base_metadata_is_current_with_retry() { - local image_ref="$1" - local metadata_version="" - - metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version") - [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]] + echo "Registry probe failed after ${attempts} attempts: ${image_ref}" >&2 + return 1 } remote_image_exists() { - local image_ref="$1" - docker manifest inspect "${image_ref}" >/dev/null 2>&1 + registry_ref_exists_with_retry manifest "$1" } use_existing_builder() { @@ -680,7 +752,7 @@ print_header() { if is_ci_base_target; then echo "Build mode: ci_base" elif is_commit_image_target; then - echo "Build mode: commit image" + echo "Build mode: build-scoped commit image" else echo "Build mode: generic" fi @@ -723,13 +795,25 @@ compute_ci_base_hash_if_needed() { if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then return 0 fi + pin_base_image + if ! is_ci_base_target; then + return 0 + fi + if [[ "${REMOTE_VLLM:-0}" != "0" ]]; then + echo "Error: content-addressed ci_base builds require REMOTE_VLLM=0" >&2 + return 1 + fi CI_BASE_CONTENT_HASH=$(compute_ci_base_content_hash) export CI_BASE_CONTENT_HASH echo "ci_base content hash: ${CI_BASE_CONTENT_HASH:0:16}..." } -should_push_stable_ci_base_tag() { +wants_stable_ci_base_tag() { + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + return 1 + fi + if [[ "${CI_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then return 0 fi @@ -740,6 +824,34 @@ should_push_stable_ci_base_tag() { [[ "${NIGHTLY:-0}" == "1" && "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]] } +trusted_ci_base_tip_matches_build() { + local branch="${CI_BASE_STABLE_BRANCH:-main}" + local build_commit="${BUILDKITE_COMMIT:-}" + local remote_tip="" + + is_trusted_ci_cache_writer || return 1 + if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Skipping ci_base stable tag: Buildkite commit is missing or invalid" >&2 + return 1 + fi + remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \ + "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }') + if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Skipping ci_base stable tag: could not resolve remote ${branch} tip" >&2 + return 1 + fi + if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then + echo "Skipping ci_base stable tag: ${branch} advanced from ${build_commit} to ${remote_tip}" >&2 + return 1 + fi +} + +should_push_stable_ci_base_tag() { + wants_stable_ci_base_tag \ + && is_trusted_ci_cache_writer \ + && trusted_ci_base_tip_matches_build +} + ci_base_tag_with_suffix() { local base_tag="$1" local suffix="$2" @@ -749,49 +861,67 @@ ci_base_tag_with_suffix() { configure_ci_base_image_refs() { local stable_tag="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}" + local metadata_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" + local scope="${CI_BASE_WRITE_SCOPE:-}" + local trusted_content_tag="" local content_tag="" - local commit_tag="" + local build_tag="" local primary_tag="" + if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then + echo "Invalid ci_base metadata version: ${metadata_version}" >&2 + return 1 + fi + + if [[ "${BUILDKITE:-false}" == "true" ]] \ + && ! is_full_git_sha "${BUILDKITE_COMMIT:-}"; then + echo "Invalid Buildkite commit for ci_base handoff: ${BUILDKITE_COMMIT:-}" >&2 + return 1 + fi + if [[ "${BUILDKITE:-false}" == "true" \ + && -z "${BUILDKITE_BUILD_ID:-}" ]]; then + echo "Buildkite build ID is required for the ci_base runtime handoff" >&2 + return 1 + fi + if [[ -n "${BUILDKITE_BUILD_ID:-}" ]]; then + build_tag=$(ci_base_tag_with_suffix \ + "${stable_tag}" "build-${BUILDKITE_BUILD_ID}") + fi + CI_BASE_IMAGE_TAG_BUILD_REF="${build_tag}" + export CI_BASE_IMAGE_TAG_BUILD_REF + if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then + if is_ci_base_target; then + echo "Error: ci_base builds require a content hash" >&2 + return 1 + fi CI_BASE_IMAGE="${CI_BASE_IMAGE:-${stable_tag}}" export CI_BASE_IMAGE return 0 fi - content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}") + trusted_content_tag=$(ci_base_tag_with_suffix \ + "${stable_tag}" "v${metadata_version}-${CI_BASE_CONTENT_HASH}") + if [[ -n "${scope}" ]]; then + content_tag=$(ci_base_tag_with_suffix \ + "${stable_tag}" "v${metadata_version}-${scope}-${CI_BASE_CONTENT_HASH}") + else + content_tag="${trusted_content_tag}" + fi CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}" - if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then - commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}") - fi - CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}" + CI_BASE_TRUSTED_CONTENT_REF="${trusted_content_tag}" + primary_tag="${build_tag:-${content_tag}}" - # *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to - # bake when that tag is not already the primary tag, avoiding duplicates. - if should_push_stable_ci_base_tag; then - primary_tag="${content_tag}" - CI_BASE_IMAGE_TAG_STABLE="${stable_tag}" - else - primary_tag="${commit_tag:-${content_tag}}" - CI_BASE_IMAGE_TAG_STABLE="" - fi + # Main writes canonical content refs. Other builds import those refs + # read-only and publish into a source-scoped preview namespace. + # A fresh CI build first publishes its unique runtime tag. Content and + # stable aliases are created only after that image passes identity checks. + CI_BASE_STABLE_PROMOTION_REF="${stable_tag}" CI_BASE_IMAGE_TAG="${primary_tag}" - if [[ "${primary_tag}" == "${content_tag}" ]]; then - CI_BASE_IMAGE_TAG_CONTENT_EXTRA="" - else - CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}" - fi - if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then - CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}" - else - CI_BASE_IMAGE_TAG_COMMIT_EXTRA="" - fi export CI_BASE_IMAGE_TAG - export CI_BASE_IMAGE_TAG_COMMIT_EXTRA - export CI_BASE_IMAGE_TAG_CONTENT_EXTRA export CI_BASE_IMAGE_TAG_CONTENT_REF - export CI_BASE_IMAGE_TAG_COMMIT_REF - export CI_BASE_IMAGE_TAG_STABLE + export CI_BASE_TRUSTED_CONTENT_REF + export CI_BASE_STABLE_PROMOTION_REF if is_ci_base_target; then IMAGE_TAG="${primary_tag}" @@ -800,29 +930,21 @@ configure_ci_base_image_refs() { export IMAGE_TAG echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}" - if [[ -n "${commit_tag}" ]]; then - if [[ "${commit_tag}" == "${primary_tag}" ]]; then - echo "ci_base commit image tag: ${commit_tag} (primary)" - else - echo "ci_base commit image tag: ${commit_tag}" - fi + if [[ -n "${build_tag}" ]]; then + echo "ci_base build image tag: ${build_tag}" fi echo "ci_base content image tag: ${content_tag}" - if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then - echo "ci_base stable alias will also be pushed: ${CI_BASE_IMAGE_TAG_STABLE}" + if wants_stable_ci_base_tag && is_trusted_ci_cache_writer; then + echo "ci_base stable alias is eligible for post-build promotion: ${stable_tag}" else echo "ci_base stable alias will not be pushed for this build" echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}" fi - set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}" - set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}" - set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}" - set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}" return 0 fi if [[ -z "${CI_BASE_IMAGE:-}" || "${CI_BASE_IMAGE}" == "${stable_tag}" ]]; then - CI_BASE_IMAGE="${primary_tag}" + CI_BASE_IMAGE="${content_tag}" export CI_BASE_IMAGE echo "Using ci_base image: ${CI_BASE_IMAGE}" else @@ -830,30 +952,71 @@ configure_ci_base_image_refs() { fi } +publish_ci_base_handoff_ref() { + local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}" + local content_ref="${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" + local digest="" + local handoff_ref="" + + is_ci_base_target || return 0 + if [[ -z "${source_ref}" || -z "${content_ref}" ]]; then + echo "Cannot publish ci_base handoff without source and content refs" >&2 + return 1 + fi + if ! digest=$(resolve_image_digest "${source_ref}"); then + echo "Could not resolve immutable ci_base handoff: ${source_ref}" >&2 + return 1 + fi + + handoff_ref="${content_ref%@*}@${digest}" + if ! confirm_remote_image_push "${handoff_ref}"; then + echo "Could not validate immutable ci_base handoff: ${handoff_ref}" >&2 + return 1 + fi + if command -v buildkite-agent >/dev/null 2>&1; then + if ! buildkite-agent meta-data set "rocm-ci-base-image" "${handoff_ref}"; then + echo "Could not publish required ci_base handoff metadata" >&2 + return 1 + fi + elif [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "buildkite-agent not found; cannot publish ci_base handoff" >&2 + return 1 + fi + echo "Published immutable ci_base handoff: ${handoff_ref}" +} + +ci_base_output_refs() { + printf '%s\n' \ + "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \ + "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" \ + | awk 'NF && !seen[$0]++' +} + ci_base_candidate_refs() { printf '%s\n' \ - "${IMAGE_TAG:-}" \ - "${CI_BASE_IMAGE_TAG:-}" \ - "${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \ - "${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \ - "${CI_BASE_IMAGE_TAG_STABLE:-}" \ + "${CI_BASE_TRUSTED_CONTENT_REF:-}" \ + "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \ + "${CI_BASE_STABLE_PROMOTION_REF:-}" \ | awk 'NF && !seen[$0]++' } find_matching_ci_base_ref() { local candidate="" - local candidate_hash="" + local candidate_digest="" + local immutable_candidate="" while IFS= read -r candidate; do [[ -n "${candidate}" ]] || continue + # Missing content refs are normal after an input change. Avoid paying + # the full digest retry budget for each definitely absent candidate. remote_image_exists "${candidate}" || continue - candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash") - if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then - if ! remote_ci_base_metadata_is_current "${candidate}"; then - echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2 - continue - fi - printf '%s\n' "${candidate}" + if ! candidate_digest=$(resolve_image_digest "${candidate}"); then + echo "Could not pin ci_base candidate: ${candidate}" >&2 + continue + fi + immutable_candidate="${candidate%@*}@${candidate_digest}" + if remote_ci_base_identity_is_current_with_retry "${immutable_candidate}"; then + printf '%s\n' "${immutable_candidate}" return 0 fi done < <(ci_base_candidate_refs) @@ -863,26 +1026,100 @@ find_matching_ci_base_ref() { refresh_ci_base_tags_from_ref() { local source_ref="$1" + local source_digest="" + local immutable_source="" local tag="" - local tag_hash="" + local tag_digest="" + + if ! source_digest=$(resolve_image_digest "${source_ref}"); then + echo "Could not resolve selected ci_base image: ${source_ref}" >&2 + return 1 + fi + immutable_source="${source_ref%@*}@${source_digest}" while IFS= read -r tag; do [[ -n "${tag}" ]] || continue [[ "${tag}" != "${source_ref}" ]] || continue - tag_hash=$(get_remote_image_label "${tag}" "vllm.ci_base.content_hash") - if [[ "${tag_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then + tag_digest="" + if remote_image_exists "${tag}"; then + tag_digest=$(resolve_image_digest "${tag}" || true) + fi + if [[ "${tag_digest}" == "${source_digest}" ]]; then echo "ci_base tag is already current: ${tag}" continue fi - echo "Updating ci_base tag ${tag} -> ${source_ref}" - docker buildx imagetools create -t "${tag}" "${source_ref}" - done < <(ci_base_candidate_refs) + echo "Updating ci_base tag ${tag} -> ${immutable_source}" + if ! docker buildx imagetools create --prefer-index=false \ + -t "${tag}" "${immutable_source}"; then + echo "Failed to update ci_base tag ${tag} from ${immutable_source}" >&2 + return 1 + fi + if ! tag_digest=$(resolve_image_digest "${tag}") \ + || [[ "${tag_digest}" != "${source_digest}" ]]; then + echo "Updated ci_base tag does not resolve to the selected digest: ${tag}" >&2 + return 1 + fi + done < <(ci_base_output_refs) +} + +promote_stable_ci_base_tag() { + local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}" + local stable_ref="${CI_BASE_STABLE_PROMOTION_REF:-}" + local digest="" + local immutable_source="" + + is_ci_base_target || return 0 + wants_stable_ci_base_tag || return 0 + if ! should_push_stable_ci_base_tag; then + echo "Skipping ci_base stable promotion: build is not the trusted current main tip" + return 0 + fi + if [[ -z "${source_ref}" || -z "${stable_ref}" ]]; then + echo "Cannot promote ci_base stable tag without source and destination refs" >&2 + return 1 + fi + if ! digest=$(resolve_image_digest "${source_ref}"); then + echo "Could not pin ci_base source before stable promotion: ${source_ref}" >&2 + return 1 + fi + immutable_source="${source_ref%@*}@${digest}" + echo "Promoting ci_base stable tag from ${immutable_source}" + if ! docker buildx imagetools create --prefer-index=false \ + -t "${stable_ref}" "${immutable_source}"; then + echo "Failed to promote ci_base stable tag" >&2 + return 1 + fi + if ! confirm_remote_image_push "${stable_ref}"; then + echo "Promoted ci_base stable tag did not become visible: ${stable_ref}" >&2 + return 1 + fi +} + +maybe_reuse_matching_ci_base_ref() { + local matching_ref="" + + matching_ref=$(find_matching_ci_base_ref) || return 1 + + echo "Found existing ci_base image with matching content hash: ${matching_ref}" + if ! refresh_ci_base_tags_from_ref "${matching_ref}"; then + echo "ci_base tag refresh failed after finding an exact image; aborting" >&2 + return 2 + fi + if ! promote_stable_ci_base_tag "${matching_ref}"; then + echo "ci_base stable promotion failed after finding an exact image; aborting" >&2 + return 2 + fi + if ! publish_ci_base_handoff_ref "${matching_ref}"; then + echo "ci_base handoff failed after finding an exact image; aborting" >&2 + return 2 + fi + echo "Content hashes match -- ci_base is current" + return 0 } maybe_skip_existing_image() { - local remote_hash="" local remote_revision="" - local matching_ref="" + local reuse_status=0 if [[ -z "${IMAGE_TAG:-}" ]]; then return 0 @@ -892,60 +1129,41 @@ maybe_skip_existing_image() { echo "FORCE_BUILD=1 set; skipping existing-image check" return 0 fi + if ! is_ci_base_target && should_upload_wheel_artifacts; then + echo "Artifact-producing targets always run for the current build" + return 0 + fi echo "--- :mag: Checking image tag" echo "Image tag: ${IMAGE_TAG}" - if ! remote_image_exists "${IMAGE_TAG}"; then - if is_ci_base_target && [[ -n "${CI_BASE_CONTENT_HASH:-}" ]]; then - matching_ref=$(find_matching_ci_base_ref || true) - if [[ -n "${matching_ref}" ]]; then - echo "Found existing ci_base image with matching content hash: ${matching_ref}" - if ! refresh_ci_base_tags_from_ref "${matching_ref}"; then - echo "ci_base tag refresh failed; rebuilding to push expected tags" - return 0 - fi - echo "Content hashes match -- ci_base is current" + if is_ci_base_target && [[ -n "${CI_BASE_CONTENT_HASH:-}" ]]; then + maybe_reuse_matching_ci_base_ref || reuse_status=$? + case "${reuse_status}" in + 0) echo "Skipping build" exit 0 - fi - fi + ;; + 1) + echo "No current ci_base image matched the expected content hash" + echo "Proceeding with build" + return 0 + ;; + *) + return "${reuse_status}" + ;; + esac + fi + + if ! remote_image_exists "${IMAGE_TAG}"; then echo "Image not found, proceeding with build" return 0 fi - IMAGE_EXISTED_BEFORE_BUILD=1 - if is_ci_base_target; then - if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then - echo "ci_base image already exists and no content hash was configured" - echo "Skipping build" - exit 0 - fi - - remote_hash=$(get_remote_image_label "${IMAGE_TAG}" "vllm.ci_base.content_hash") - if [[ -n "${remote_hash}" ]]; then - echo "Remote ci_base content hash: ${remote_hash:0:16}..." - if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then - if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then - echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata" - return 0 - fi - if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then - echo "ci_base tag refresh failed; rebuilding to push expected tags" - return 0 - fi - echo "Content hashes match -- ci_base is current" - echo "Skipping build" - exit 0 - fi - - echo "Content hashes differ -- ci_base is stale, rebuilding" - return 0 - fi - - echo "Remote ci_base has no content-hash label; rebuilding to add one" - return 0 + echo "ci_base image already exists and no content hash was configured" + echo "Skipping build" + exit 0 fi if is_commit_image_target; then @@ -957,12 +1175,6 @@ maybe_skip_existing_image() { return 0 fi - if should_upload_wheel_artifacts; then - echo "Commit image already exists: ${IMAGE_TAG}" - echo "Continuing build because this target uploads per-build ROCm artifacts" - return 0 - fi - echo "Commit image already exists: ${IMAGE_TAG}" echo "Skipping build" exit 0 @@ -1031,6 +1243,18 @@ setup_builder() { docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls } +validate_cache_branch_tag() { + local name="$1" + local value="$2" + + if [[ -n "${value}" \ + && (! "${value}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ \ + || ${#value} -gt ${ROCM_CACHE_BRANCH_TAG_MAX_LEN}) ]]; then + echo "Invalid ${name}; expected a Docker tag component of at most ${ROCM_CACHE_BRANCH_TAG_MAX_LEN} characters" >&2 + return 1 + fi +} + prepare_git_cache_metadata() { local cache_branch_name="" local cache_base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" @@ -1038,6 +1262,11 @@ prepare_git_cache_metadata() { local target_repo_url="" local merge_base_ref="" + if is_ci_base_target; then + echo "Skipping commit-cache ancestry lookup for content-addressed ci_base" + return 0 + fi + if [[ -z "${PARENT_COMMIT:-}" || -z "${VLLM_MERGE_BASE_COMMIT:-}" ]] \ && git rev-parse --is-shallow-repository 2>/dev/null | grep -q "true"; then echo "Shallow clone detected - deepening for cache key computation" @@ -1090,6 +1319,11 @@ prepare_git_cache_metadata() { echo "Using provided ROCM_CACHE_UPSTREAM_BRANCH_TAG: ${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" fi + validate_cache_branch_tag \ + ROCM_CACHE_BRANCH_TAG "${ROCM_CACHE_BRANCH_TAG:-}" + validate_cache_branch_tag \ + ROCM_CACHE_UPSTREAM_BRANCH_TAG "${ROCM_CACHE_UPSTREAM_BRANCH_TAG:-}" + if [[ -z "${VLLM_MERGE_BASE_COMMIT:-}" ]]; then target_repo_url=$(get_buildkite_target_repo_url) merge_base_ref="refs/remotes/vllm-cache-upstream/${cache_base_branch}" @@ -1119,13 +1353,15 @@ ci_base_metadata_pairs() { local content_files_hash="" local base_image="" local base_image_digest="" - local git_branch="" local -a content_paths=() local -a content_args=() read -r -a content_paths <<< "${content_files}" if [[ ${#content_paths[@]} -gt 0 ]]; then - content_files_hash=$(compute_content_hash "${content_paths[@]}") + if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then + echo "Failed to hash ci_base metadata content files" >&2 + return 1 + fi fi mapfile -t content_args < <( get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}" @@ -1133,9 +1369,11 @@ ci_base_metadata_pairs() { base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE") if [[ -n "${base_image}" ]]; then - base_image_digest=$(resolve_image_digest "${base_image}") + if ! base_image_digest=$(resolve_image_digest "${base_image}"); then + echo "Failed to resolve ci_base metadata digest for ${base_image}" >&2 + return 1 + fi fi - git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}" metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}" @@ -1144,21 +1382,16 @@ ci_base_metadata_pairs() { metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")" metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}" metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}" - metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}" - metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}" - metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}" - metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}" - metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}" - metadata_pair "vllm.ci_base.git_branch" "${git_branch}" - metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}" - metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}" - metadata_pair "vllm.rocm.base_image" "${base_image}" + # The parent identity is digest-only: mutable aliases that resolve to the + # same image must produce byte-identical canonical metadata. + metadata_pair "vllm.rocm.base_image" "${base_image_digest}" metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}" metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}" metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")" metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")" metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")" + metadata_pair "vllm.rocm.max_jobs" "$(resolve_dockerfile_arg_value "${dockerfile}" "max_jobs")" metadata_pair "vllm.rocm.nixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_REPO")" metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}" metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")" @@ -1172,22 +1405,18 @@ ci_base_metadata_pairs() { metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}" metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}" metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}" - - metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}" - metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}" } write_ci_base_metadata_annotations() { local metadata="$1" local key="" local value="" - local annotation="" [[ -n "${metadata}" ]] || return 0 while IFS=$'\t' read -r key value; do [[ -n "${key}" && -n "${value}" ]] || continue - annotation="manifest:${key}=${value}" - printf ' "%s",\n' "$(hcl_escape_string "${annotation}")" + printf ' "%s",\n' \ + "$(hcl_escape_string "manifest:${key}=${value}")" done <<< "${metadata}" } @@ -1277,23 +1506,16 @@ uses_rocm_rust_cache() { compute_rocm_csrc_content_hash() { local bake_dir="" local dockerfile_rocm="" - local -a content_paths=( - "requirements/common.txt" - "requirements/rocm.txt" - "pyproject.toml" - "setup.py" - "CMakeLists.txt" - "cmake" - "csrc" - "vllm/envs.py" - "vllm/__init__.py" - ) + local content_files="${ROCM_CSRC_CONTENT_FILES:-${DEFAULT_ROCM_CSRC_CONTENT_FILES}}" + local stages="${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}" + local -a content_paths=() local -a content_args=() bake_dir=$(dirname "${VLLM_BAKE_FILE}") dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + read -r -a content_paths <<< "${content_files}" mapfile -t content_args < <( - get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}" + get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}" ) { @@ -1301,9 +1523,9 @@ compute_rocm_csrc_content_hash() { printf 'dockerfile:%s\n' "${dockerfile_rocm}" printf 'resolved-build-args:\n' hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}" - printf 'dockerfile-stages:base csrc-build\n' + printf 'dockerfile-stages:%s\n' "${stages}" if [[ -f "${dockerfile_rocm}" ]]; then - hash_dockerfile_stages "${dockerfile_rocm}" "base csrc-build" + hash_dockerfile_stages "${dockerfile_rocm}" "${stages}" else printf 'missing:%s\n' "${dockerfile_rocm}" fi @@ -1312,14 +1534,20 @@ compute_rocm_csrc_content_hash() { compute_rocm_csrc_content_hash_if_needed() { local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}" + local write_scope="${CI_BASE_WRITE_SCOPE:-}" if [[ "${ROCM_CSRC_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_csrc_cache; then return 0 fi ROCM_CSRC_CONTENT_HASH=$(compute_rocm_csrc_content_hash) - ROCM_CSRC_CONTENT_CACHE_REF="${cache_repo}:csrc-rocm-input-${ROCM_CSRC_CONTENT_HASH}" + ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:csrc-rocm-input-${ROCM_CSRC_CONTENT_HASH}" + ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}" + if [[ -n "${write_scope}" ]]; then + ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_CONTENT_CACHE_REF}-${write_scope}" + fi export ROCM_CSRC_CONTENT_HASH + export ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF export ROCM_CSRC_CONTENT_CACHE_REF echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}" } @@ -1327,23 +1555,17 @@ compute_rocm_csrc_content_hash_if_needed() { compute_rocm_rust_content_hash() { local bake_dir="" local dockerfile_rocm="" - local -a content_paths=( - "requirements/build/rust.txt" - "rust/Cargo.lock" - "rust/Cargo.toml" - "rust/proto" - "rust/src" - "rust-toolchain.toml" - "tools/build_rust.py" - "tools/install_protoc.sh" - "build_rust.sh" - ) + local content_files="${ROCM_RUST_CONTENT_FILES:-${DEFAULT_ROCM_RUST_CONTENT_FILES}}" + local stages="${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}" + local -a content_paths=() local -a content_args=() bake_dir=$(dirname "${VLLM_BAKE_FILE}") dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + read -r -a content_paths <<< "${content_files}" mapfile -t content_args < <( - get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}" + get_content_arg_names \ + "${dockerfile_rocm}" "${stages}" "${ROCM_RUST_CONTENT_ARGS:-}" ) { @@ -1351,9 +1573,9 @@ compute_rocm_rust_content_hash() { printf 'dockerfile:%s\n' "${dockerfile_rocm}" printf 'resolved-build-args:\n' hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}" - printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n' + printf 'dockerfile-stages:%s\n' "${stages}" if [[ -f "${dockerfile_rocm}" ]]; then - hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" + hash_dockerfile_stages "${dockerfile_rocm}" "${stages}" else printf 'missing:%s\n' "${dockerfile_rocm}" fi @@ -1362,14 +1584,20 @@ compute_rocm_rust_content_hash() { compute_rocm_rust_content_hash_if_needed() { local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}" + local write_scope="${CI_BASE_WRITE_SCOPE:-}" if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then return 0 fi ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash) - ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}" + ROCM_RUST_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}" + ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}" + if [[ -n "${write_scope}" ]]; then + ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_CONTENT_CACHE_REF}-${write_scope}" + fi export ROCM_RUST_CONTENT_HASH + export ROCM_RUST_TRUSTED_CONTENT_CACHE_REF export ROCM_RUST_CONTENT_CACHE_REF echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}" } @@ -1430,8 +1658,14 @@ write_rocm_build_arg_override() { "${dockerfile_rocm}" \ "${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \ "${CI_BASE_CONTENT_ARGS:-}" - get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}" - get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}" + get_content_arg_names \ + "${dockerfile_rocm}" \ + "${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}" \ + "${ROCM_CSRC_CONTENT_ARGS:-}" + get_content_arg_names \ + "${dockerfile_rocm}" \ + "${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}" \ + "${ROCM_RUST_CONTENT_ARGS:-}" } | awk 'NF && !seen[$0]++' ) @@ -1497,6 +1731,7 @@ validate_content_cache_export_mode() { should_export_content_cache_ref() { local cache_ref="$1" local cache_name="$2" + local trusted_ref="${3:-${cache_ref}}" local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}" case "${mode}" in @@ -1509,8 +1744,13 @@ should_export_content_cache_ref() { return 1 ;; missing|"") - if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then - echo "${cache_name} content cache exists; not re-exporting ${cache_ref}" + if remote_image_exists "${trusted_ref}"; then + echo "${cache_name} trusted content cache exists; not exporting ${cache_ref}" + return 1 + fi + if [[ "${cache_ref}" != "${trusted_ref}" ]] \ + && remote_image_exists "${cache_ref}"; then + echo "${cache_name} scoped content cache exists; not re-exporting ${cache_ref}" return 1 fi echo "${cache_name} content cache missing; will export ${cache_ref}" @@ -1555,8 +1795,18 @@ write_rocm_cache_override() { echo "ROCm final image cache export mode: ${rocm_cache_to_mode}" if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then - csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}") - if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then + csrc_content_cache_from+=( + "type=registry,ref=${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}" + ) + if [[ "${ROCM_CSRC_CONTENT_CACHE_REF}" != \ + "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}" ]]; then + csrc_content_cache_from+=( + "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}" + ) + fi + if should_export_content_cache_ref \ + "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc" \ + "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"; then csrc_cache_to+=( "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true" ) @@ -1566,8 +1816,18 @@ write_rocm_cache_override() { fi if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then - rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}") - if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then + rust_content_cache_from+=( + "type=registry,ref=${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}" + ) + if [[ "${ROCM_RUST_CONTENT_CACHE_REF}" != \ + "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}" ]]; then + rust_content_cache_from+=( + "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}" + ) + fi + if should_export_content_cache_ref \ + "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust" \ + "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"; then rust_cache_to+=( "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true" ) @@ -1772,7 +2032,7 @@ compose_stage_cache_material() { dependency_cache_ref_exists() { local cache_ref="$1" - docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1 + registry_ref_exists_with_retry imagetools "${cache_ref}" } dependency_cache_ref_for_target() { @@ -1914,54 +2174,18 @@ print_bake_config() { confirm_remote_image_push() { local image_ref="$1" - local remote_hash="" - local remote_revision="" - if ! remote_image_exists "${image_ref}"; then - return 1 + if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then + remote_image_exists "${image_ref}" + return fi - if is_ci_base_target; then - if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then - return 0 - fi - - remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash") - if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then - if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then - return 0 - fi - - echo "Remote image exists with the expected ci_base content hash but stale metadata." - echo " expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" - return 1 - fi - - echo "Remote image exists but does not have the expected ci_base content hash." - echo " expected: ${CI_BASE_CONTENT_HASH:0:16}..." - echo " found: ${remote_hash:0:16}..." - return 1 + if remote_ci_base_identity_is_current_with_retry "${image_ref}"; then + return 0 fi - if is_commit_image_target; then - remote_revision=$(get_remote_image_label_with_retry "${image_ref}" "org.opencontainers.image.revision") - if [[ -n "${remote_revision}" && "${remote_revision}" == "${BUILDKITE_COMMIT}" ]]; then - return 0 - fi - - if [[ -z "${remote_revision}" \ - && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \ - && image_tag_is_commit_scoped; then - echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label." - return 0 - fi - - echo "Remote image exists but revision label does not match ${BUILDKITE_COMMIT}." - echo " found revision: ${remote_revision:-}" - return 1 - fi - - return 0 + echo "Remote image does not have the expected complete ci_base identity." + return 1 } verify_dependency_cache_ref() { @@ -2017,64 +2241,27 @@ seed_dependency_caches_if_needed() { done } -annotate_cache_export_warning() { - local build_rc="$1" - - if ! command -v buildkite-agent >/dev/null 2>&1; then - return 0 - fi - - buildkite-agent annotate \ - --style warning \ - --context "cache-export-warning" \ - "### :warning: Docker cache export failed (non-fatal) - -Image was pushed successfully: \`${IMAGE_TAG}\` - -The BuildKit build returned exit code ${build_rc}, but the expected image -is present in the registry. Treating this as a registry cache export failure -so tests can continue with the pushed image." 2>/dev/null || true -} - run_bake() { - local build_rc=0 + local confirmation_ref="${IMAGE_TAG:-}" + + if is_ci_base_target && [[ -n "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" ]]; then + confirmation_ref="${CI_BASE_IMAGE_TAG_BUILD_REF}" + fi echo "--- :docker: Building ${TARGET}" docker buildx bake \ "${BAKE_FILES[@]}" \ --progress "${BUILDKIT_PROGRESS:-plain}" \ - "${BAKE_TARGETS[@]}" || build_rc=$? + "${BAKE_TARGETS[@]}" - if [[ ${build_rc} -eq 0 ]]; then - echo "--- :white_check_mark: Build complete" - return 0 - fi - - echo "" - echo "WARNING: docker buildx bake exited with code ${build_rc}" - - if [[ -n "${IMAGE_TAG:-}" ]]; then - echo "Checking if image was pushed successfully..." - if confirm_remote_image_push "${IMAGE_TAG}"; then - echo "" - echo "WARNING: Build reported failure (rc=${build_rc}) but the" - echo " image was pushed successfully: ${IMAGE_TAG}" - echo "" - echo " Treating this as a non-fatal registry cache export failure." - echo " The image is usable, but registry cache may be cold on the next build." - echo "" - annotate_cache_export_warning "${build_rc}" - echo "--- :white_check_mark: Build complete" - return 0 + if is_ci_base_target; then + if ! confirm_remote_image_push "${confirmation_ref}"; then + echo "Fresh ci_base image failed identity validation: ${confirmation_ref}" >&2 + return 1 fi - - echo "" - echo "ERROR: Build failed and image was NOT confirmed: ${IMAGE_TAG}" - echo " This is a real build failure, not a cache export warning." - echo "" + refresh_ci_base_tags_from_ref "${confirmation_ref}" fi - - return "${build_rc}" + echo "--- :white_check_mark: Build complete" } upload_wheel_artifacts_if_present() { @@ -2082,7 +2269,10 @@ upload_wheel_artifacts_if_present() { local artifact_dir="artifacts/vllm-rocm-install" local archive_name="vllm-rocm-install.tar.gz" local metadata_dir="${wheel_dir}/.vllm-ci-artifact" + local build_base_digest="" + local expected_native_base_image="" local native_base_image="" + local native_base_digest="" local whl="" local whl_name="" local -a wheels=() @@ -2100,11 +2290,35 @@ upload_wheel_artifacts_if_present() { fi whl="${wheels[0]}" whl_name=$(basename "${whl}") - native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}" + native_base_image="${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE:-}}" if [[ -z "${native_base_image}" ]]; then echo "Native ROCm artifact requires a ci_base image reference" >&2 return 1 fi + if [[ "${BUILDKITE:-false}" == "true" ]]; then + expected_native_base_image=$(ci_base_tag_with_suffix \ + "${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}" \ + "build-${BUILDKITE_BUILD_ID:-}") + if [[ "${native_base_image}" != "${expected_native_base_image}" ]]; then + echo "Native ROCm artifact requires the exact ci_base build handoff: ${native_base_image}" >&2 + return 1 + fi + if [[ ! "${CI_BASE_IMAGE:-}" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "ROCm artifact build base must be digest-pinned: ${CI_BASE_IMAGE:-}" >&2 + return 1 + fi + build_base_digest="${CI_BASE_IMAGE##*@}" + if ! native_base_digest=$(resolve_image_digest "${native_base_image}"); then + echo "Could not resolve native ci_base handoff: ${native_base_image}" >&2 + return 1 + fi + if [[ "${native_base_digest}" != "${build_base_digest}" ]]; then + echo "Native ci_base handoff does not match the artifact build base" >&2 + echo " native: ${native_base_image}@${native_base_digest}" >&2 + echo " build: ${CI_BASE_IMAGE}" >&2 + return 1 + fi + fi echo "--- :package: Uploading ROCm vLLM install artifact" rm -rf "${artifact_dir}" "${metadata_dir}" @@ -2141,8 +2355,10 @@ upload_wheel_artifacts_if_present() { main() { init_config "$@" + configure_ci_base_write_scope print_header validate_inputs + normalize_ci_worktree_modes load_ci_hcl init_bake_files compute_ci_base_hash_if_needed @@ -2170,7 +2386,11 @@ main() { fi seed_dependency_caches_if_needed run_bake + promote_stable_ci_base_tag + publish_ci_base_handoff_ref upload_wheel_artifacts_if_present } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 8b5afe5a9b..635b6f1de5 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -36,6 +36,19 @@ set -o pipefail : "${ROCM_DOCKER_TTY:=1}" : "${PYTHONFAULTHANDLER:=1}" : "${PYTEST_TIMEOUT:=2400}" + +# Preserve runner-only values before clear_ci_orchestration_env removes them +# from the environment inherited by the test process. +amd_diagnostics_expected_gpu_count="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" +amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}" +amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}" +amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}" +amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}" +amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}" +amd_diagnostics_collected=0 +amd_diagnostics_probe_budget_seconds=25 +amd_diagnostics_command_timeout_seconds=5 +amd_diagnostics_upload_timeout_seconds=20 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" fi @@ -84,9 +97,11 @@ clear_ci_orchestration_env() { VLLM_CI_BASE_IMAGE \ VLLM_CI_FALLBACK_IMAGE \ VLLM_CI_DOCKER_DISABLED \ + VLLM_CI_DIAGNOSTICS_DIR \ VLLM_CI_ARTIFACT_GLOB \ VLLM_CI_ARTIFACT_CHECKSUM_GLOB \ VLLM_CI_EXPECTED_GPU_COUNT \ + VLLM_CI_K8S_NODE_NAME \ VLLM_CI_USE_ARTIFACTS \ VLLM_CI_RESULTS_ROOT \ VLLM_ALLOW_DEPRECATED_BEAM_SEARCH @@ -350,6 +365,11 @@ prepare_native_workspace() { echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2 return 1 fi + if [[ -z "${BUILDKITE_BUILD_ID:-}" \ + || "${recorded_base}" != *":ci_base-build-${BUILDKITE_BUILD_ID}" ]]; then + echo "ROCm artifact base is not scoped to Buildkite build ${BUILDKITE_BUILD_ID:-unset}: ${recorded_base}" >&2 + return 1 + fi if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2 return 1 @@ -495,12 +515,220 @@ is_multi_node() { return 1 } +run_amd_diagnostic() { + local log_file=$1 + local probe_deadline=$2 + shift 2 + local command_status=0 + local command_timeout_seconds=${amd_diagnostics_command_timeout_seconds} + local remaining_seconds=$((probe_deadline - SECONDS)) + + { + printf '\n$' + printf ' %q' "$@" + printf '\n' + } | tee -a "${log_file}" + + if [[ "${remaining_seconds}" -le 0 ]]; then + echo "[diagnostic probe budget exhausted; command skipped]" \ + | tee -a "${log_file}" + return 0 + fi + if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then + command_timeout_seconds=${remaining_seconds} + fi + + timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \ + | tee -a "${log_file}" + command_status=${PIPESTATUS[0]} + if [[ "${command_status}" -ne 0 ]]; then + echo "[diagnostic command exited with status ${command_status}]" \ + | tee -a "${log_file}" + fi + return 0 +} + +collect_rocm_failure_diagnostics() { + local exit_code=$1 + local job_id="${BUILDKITE_JOB_ID:-local}" + local retry_count="${BUILDKITE_RETRY_COUNT:-0}" + local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}" + local diagnostics_relative_path="" + local diagnostics_path="" + local upload_root="${amd_diagnostics_checkout_root}" + local upload_path="" + local exit_signal="" + local probe_deadline=0 + + if [[ "${amd_diagnostics_collected}" == "1" ]]; then + return 0 + fi + amd_diagnostics_collected=1 + + if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then + echo "Skipping AMD GPU diagnostics for a CPU-only job." + return 0 + fi + + if ! command -v timeout >/dev/null 2>&1; then + echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job." + return 0 + fi + + job_id="${job_id//[^A-Za-z0-9_.-]/_}" + retry_count="${retry_count//[^A-Za-z0-9_.-]/_}" + parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}" + + # Buildkite keeps the supplied artifact path. Restrict the configurable + # directory to a checkout-relative path so the UI shows a clean artifact key. + if [[ -z "${amd_diagnostics_dir}" \ + || "${amd_diagnostics_dir}" == /* \ + || "${amd_diagnostics_dir}" == *".."* ]]; then + echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR" + amd_diagnostics_dir="artifacts/amd-gpu-diagnostics" + fi + diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log" + diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}" + upload_path="${diagnostics_relative_path}" + + if ! mkdir -p "$(dirname "${diagnostics_path}")" \ + || ! : > "${diagnostics_path}"; then + diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || { + echo "WARNING: unable to create an AMD GPU diagnostics log." + return 0 + } + upload_root=$(dirname "${diagnostics_path}") + upload_path=$(basename "${diagnostics_path}") + fi + + if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then + exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true) + fi + + { + echo "AMD GPU failure diagnostics" + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "exit_code=${exit_code}" + echo "exit_signal=${exit_signal:-none}" + echo "build_url=${BUILDKITE_BUILD_URL:-unknown}" + echo "job_id=${BUILDKITE_JOB_ID:-unknown}" + echo "step_key=${BUILDKITE_STEP_KEY:-unknown}" + echo "step_label=${BUILDKITE_LABEL:-unknown}" + echo "test_group=${amd_diagnostics_test_group}" + echo "retry_count=${retry_count}" + echo "parallel_job=${parallel_job}" + echo "execution_mode=${amd_diagnostics_execution_mode}" + echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}" + echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}" + echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}" + echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}" + echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}" + echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}" + echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)" + } > "${diagnostics_path}" + + echo "--- :rotating_light: AMD GPU failure diagnostics" + cat "${diagnostics_path}" + + probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds)) + + # Collect the most useful evidence first so auxiliary probes cannot consume + # the shared deadline before GPU health is captured. + echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}" + if command -v amd-smi >/dev/null 2>&1; then + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version + # Bus data identifies a card within the public node without publishing its + # persistent UUID, serial number, or process list. + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all + elif command -v rocm-smi >/dev/null 2>&1; then + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ + --showbus --showreplaycount --showrasinfo --showpagesinfo + else + echo "Neither amd-smi nor rocm-smi is available." \ + | tee -a "${diagnostics_path}" + fi + + echo "Node resource diagnostics (runner-visible scope)" \ + | tee -a "${diagnostics_path}" + if command -v df >/dev/null 2>&1; then + echo "checkout_filesystem:" | tee -a "${diagnostics_path}" + if [[ -d "${amd_diagnostics_checkout_root}" ]]; then + ( + cd "${amd_diagnostics_checkout_root}" || exit 0 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \ + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \ + -- . + ) + else + echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}" + fi + echo "tmp_filesystem:" | tee -a "${diagnostics_path}" + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \ + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \ + -- /tmp + fi + if command -v free >/dev/null 2>&1; then + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h + fi + if [[ -r /sys/fs/cgroup/memory.events ]]; then + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \ + /sys/fs/cgroup/memory.events + fi + + echo "Unauthenticated outer-runner network reachability diagnostics" \ + | tee -a "${diagnostics_path}" + if command -v curl >/dev/null 2>&1; then + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ + --silent --location --proto '=https' --proto-redir '=https' \ + --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ + --write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ + https://huggingface.co/api/models/gpt2 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ + --silent --location --proto '=https' --proto-redir '=https' \ + --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ + --write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ + 'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack' + else + echo "curl unavailable; reachability probes skipped." \ + | tee -a "${diagnostics_path}" + fi + + if command -v buildkite-agent >/dev/null 2>&1 \ + && [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then + if ( + cd "${upload_root}" \ + && BUILDKITE_AGENT_DEBUG=false \ + BUILDKITE_AGENT_DEBUG_HTTP=false \ + BUILDKITE_AGENT_TRACE_HTTP=false \ + BUILDKITE_AGENT_LOG_LEVEL=error \ + timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \ + buildkite-agent artifact upload "${upload_path}" \ + >/dev/null 2>&1 + ); then + echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}" + else + echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}" + fi + else + echo "Buildkite agent unavailable; diagnostics artifact was not uploaded." + fi + + return 0 +} + handle_pytest_exit() { local exit_code=$1 if [ "$exit_code" -eq 5 ]; then echo "Pytest exit code 5 (no tests collected) - treating as success." exit 0 fi + if [[ "${exit_code}" -ne 0 ]]; then + collect_rocm_failure_diagnostics "${exit_code}" + fi exit "$exit_code" } @@ -659,6 +887,19 @@ re_quote_pytest_markers() { echo "${output% }" } +# shellcheck disable=SC2317 # Called indirectly by the EXIT trap. +handle_amd_runner_exit() { + local exit_code=${1:-$?} + if [[ "${exit_code}" -ne 0 ]]; then + collect_rocm_failure_diagnostics "${exit_code}" + fi + return 0 +} + +# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup +# traps below replace this trap but call the same handler before cleaning up. +trap handle_amd_runner_exit EXIT + ############################################################################### # Main ############################################################################### @@ -667,7 +908,10 @@ if is_native_runtime; then echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" artifact_work_dir="" + # shellcheck disable=SC2317 # Called indirectly by the EXIT trap. cleanup_native_workspace() { + local exit_code=$? + handle_amd_runner_exit "${exit_code}" if [[ -n "${artifact_work_dir}" ]]; then rm -rf "${artifact_work_dir}" fi @@ -727,7 +971,10 @@ image_name="${VLLM_CI_FALLBACK_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:-local}}" artifact_work_dir="" container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" +# shellcheck disable=SC2317 # Called indirectly by the EXIT trap. remove_docker_container() { + local exit_code=$? + handle_amd_runner_exit "${exit_code}" if docker container inspect "${container_name}" >/dev/null 2>&1; then docker rm -f "${container_name}" || true fi diff --git a/.buildkite/scripts/rocm/build-ci-base.sh b/.buildkite/scripts/rocm/build-ci-base.sh index 23d17e17b4..7d4eb381bc 100755 --- a/.buildkite/scripts/rocm/build-ci-base.sh +++ b/.buildkite/scripts/rocm/build-ci-base.sh @@ -12,17 +12,33 @@ metadata_get() { main() { local base_refreshed="" + local selected_base="" + + # The content identity below describes this checkout. Do not let legacy + # external pipeline settings switch the build to a remote source tree. + export REMOTE_VLLM=0 + unset VLLM_BRANCH base_refreshed="$(metadata_get rocm-base-refresh)" + selected_base="$(metadata_get rocm-base-image)" + if [[ -n "${selected_base}" ]]; then + if [[ ! "${selected_base}" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "Selected ROCm base handoff is not digest-pinned: ${selected_base}" >&2 + return 1 + fi + export BASE_IMAGE="${selected_base}" + echo "Using selected ROCm base image for ci_base: ${BASE_IMAGE}" + elif [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "Required ROCm base handoff metadata is missing" >&2 + return 1 + fi + if [[ "${base_refreshed}" == "1" ]]; then - export BASE_IMAGE export CI_BASE_PUSH_STABLE_TAG - BASE_IMAGE="$(metadata_get rocm-base-image)" CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)" CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}" - echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}" echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}" fi diff --git a/.buildkite/scripts/rocm/build-test-image.sh b/.buildkite/scripts/rocm/build-test-image.sh index 9feb6c0924..69c4f77d20 100755 --- a/.buildkite/scripts/rocm/build-test-image.sh +++ b/.buildkite/scripts/rocm/build-test-image.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Build the ROCm CI test image or wheel artifact. # -# When Dockerfile.rocm_base changes, always build the full image so downstream -# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain. +# When base selection changes, build the full image so downstream ROCm tests +# validate the selected base -> ci_base -> CI image chain. set -euo pipefail @@ -13,46 +13,55 @@ metadata_get() { fi } -use_ci_base_if_present() { - local ci_base_image="" +load_digest_handoff() { + local metadata_key="$1" + local env_name="$2" + local description="$3" + local image_ref="" - ci_base_image="$(metadata_get rocm-ci-base-image)" - if [[ -z "${ci_base_image}" ]]; then + image_ref="$(metadata_get "${metadata_key}")" + if [[ -z "${image_ref}" ]]; then + return 1 + fi + if [[ ! "${image_ref}" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "${description} is not digest-pinned: ${image_ref}" >&2 return 1 fi - export CI_BASE_IMAGE="${ci_base_image}" - echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}" -} - -use_refreshed_base_if_present() { - local base_refreshed="" - - base_refreshed="$(metadata_get rocm-base-refresh)" - if [[ "${base_refreshed}" != "1" ]]; then - return 1 - fi - - export BASE_IMAGE - export IMAGE_TAG_LATEST - - BASE_IMAGE="$(metadata_get rocm-base-image)" - IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)" - - echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}" - if [[ -n "${IMAGE_TAG_LATEST}" ]]; then - echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}" - fi - - return 0 + printf -v "${env_name}" '%s' "${image_ref}" + export "${env_name?}" + echo "Using ${description}: ${image_ref}" } main() { local base_refreshed=0 - use_ci_base_if_present || true + # This job always builds the checked-out commit. Some externally generated + # pipeline templates still inject remote-fetch settings; do not let those + # settings make the source identity commit-specific or bypass local edits. + export REMOTE_VLLM=0 + unset VLLM_BRANCH - if use_refreshed_base_if_present; then + if ! load_digest_handoff \ + rocm-ci-base-image CI_BASE_IMAGE "ROCm ci_base handoff"; then + if [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "Required ROCm ci_base handoff metadata is missing or invalid" >&2 + return 1 + fi + echo "No ROCm ci_base handoff metadata found; using the local default" + fi + + if ! load_digest_handoff \ + rocm-base-image BASE_IMAGE "ROCm base handoff"; then + if [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "Required ROCm base handoff metadata is missing or invalid" >&2 + return 1 + fi + echo "No ROCm base handoff metadata found; using the local default" + fi + + if [[ "$(metadata_get rocm-base-refresh)" == "1" ]]; then + echo "The selected ROCm base differs from the current stable base" base_refreshed=1 fi diff --git a/.buildkite/scripts/rocm/refresh-base-image.sh b/.buildkite/scripts/rocm/refresh-base-image.sh index 06e3e80c96..23d10492a9 100755 --- a/.buildkite/scripts/rocm/refresh-base-image.sh +++ b/.buildkite/scripts/rocm/refresh-base-image.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash -# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes. -# -# Normal AMD CI builds should not pay for this path. The script no-ops unless -# docker/Dockerfile.rocm_base changed relative to the branch base, the previous -# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set. +# Select the content-addressed ROCm base for this checkout, building it only +# when the exact content ref is absent. set -euo pipefail DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}" BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}" -CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}" +CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}" BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}" -DEFAULT_ROCM_BASE_METADATA_VERSION="1" +DEFAULT_ROCM_BASE_METADATA_VERSION="2" DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}" -DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE" + +ROCM_BASE_LAYER_CACHE_REF="" +ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main" +ROCM_BASE_STABLE_TAG_UPDATED=0 +declare -a ROCM_BASE_CACHE_ARGS=() metadata_set() { local key="$1" @@ -21,7 +22,10 @@ metadata_set() { [[ -n "${value}" ]] || return 0 if command -v buildkite-agent >/dev/null 2>&1; then - buildkite-agent meta-data set "${key}" "${value}" || true + buildkite-agent meta-data set "${key}" "${value}" + elif [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "buildkite-agent is required to publish ROCm base metadata" >&2 + return 1 fi } @@ -56,6 +60,82 @@ tag_component() { clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}" } +normalize_repo_slug() { + local repo_slug="${1:-}" + + repo_slug="${repo_slug%/}" + repo_slug="${repo_slug%.git}" + repo_slug="${repo_slug#https://github.com/}" + repo_slug="${repo_slug#http://github.com/}" + repo_slug="${repo_slug#ssh://git@github.com/}" + repo_slug="${repo_slug#git@github.com:}" + repo_slug="${repo_slug#github.com/}" + printf '%s\n' "${repo_slug}" +} + +is_trusted_main_build() { + local actual_repo="" + local trusted_repo="" + + [[ "${BUILDKITE:-false}" == "true" ]] || return 1 + [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1 + [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] || return 1 + actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}") + trusted_repo=$(normalize_repo_slug \ + "${ROCM_BASE_STABLE_REPO_SLUG:-vllm-project/vllm}") + [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]] +} + +rocm_base_layer_cache_scope() { + local pull_request="${BUILDKITE_PULL_REQUEST:-false}" + local branch="${BUILDKITE_PULL_REQUEST_HEAD_BRANCH:-${BUILDKITE_BRANCH:-local}}" + local identity="" + local repo_slug="" + + if is_trusted_main_build; then + printf 'main\n' + return 0 + fi + if [[ "${pull_request}" != "false" && -n "${pull_request}" ]]; then + repo_slug=$(normalize_repo_slug "${BUILDKITE_REPO:-local}") + identity=$(printf '%s\n' "${repo_slug:-local}" | sha256sum | cut -c1-12) + printf 'pr-%s-%s\n' \ + "$(tag_component "${pull_request}" 32)" "${identity}" + return 0 + fi + + identity=$(printf '%s\n%s\n' "${BUILDKITE_REPO:-local}" "${branch}" \ + | sha256sum | cut -c1-12) + printf 'preview-%s-%s\n' "$(tag_component "${branch}" 24)" "${identity}" +} + +configure_rocm_base_layer_cache() { + local scope="" + + ROCM_BASE_CACHE_ARGS=() + if [[ "${ROCM_BASE_NO_CACHE:-0}" == "1" ]]; then + ROCM_BASE_CACHE_ARGS+=(--no-cache) + ROCM_BASE_LAYER_CACHE_REF="disabled" + return 0 + fi + + scope=$(rocm_base_layer_cache_scope) + ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}" + ROCM_BASE_CACHE_ARGS+=( + --cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}" + ) + if [[ "${ROCM_BASE_LAYER_CACHE_REF}" != \ + "${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" ]]; then + ROCM_BASE_CACHE_ARGS+=( + --cache-from "type=registry,ref=${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" + ) + fi + ROCM_BASE_CACHE_ARGS+=( + --cache-to \ + "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF},mode=max,ignore-error=true" + ) +} + extract_arg_default() { local arg_name="$1" @@ -65,41 +145,123 @@ extract_arg_default() { resolve_image_digest() { local image_ref="$1" + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}" + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}" + local attempt=0 + local output="" + local digest="" + local status=0 - docker buildx imagetools inspect "${image_ref}" 2>/dev/null \ - | sed -n -E 's/^Digest:[[:space:]]+//p' \ - | head -1 || true -} + if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" + return 0 + fi + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid image digest retry configuration" >&2 + return 1 + fi -resolve_rocm_base_arg_value() { - local arg_name="$1" - local use_sccache="$2" - - case "${arg_name}" in - USE_SCCACHE) - printf '%s\n' "${use_sccache}" - ;; - *) - extract_arg_default "${arg_name}" - ;; - esac -} - -hash_rocm_base_arg_values() { - local use_sccache="$1" - local base_image_digest="$2" - local arg_name="" - local arg_value="" - shift 2 || true - - for arg_name in "$@"; do - [[ -n "${arg_name}" ]] || continue - arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}") - printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-}" - if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then - printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}" + for ((attempt = 1; attempt <= attempts; attempt++)); do + status=0 + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$? + digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}") + if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + printf '%s\n' "${digest}" + return 0 + fi + if ((attempt < attempts)); then + printf \ + 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \ + "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2 + sleep "${delay_secs}" fi done + + printf 'Failed to resolve digest for %s (status %d)\n%s\n' \ + "${image_ref}" "${status}" "${output:-}" >&2 + return 1 +} + +trusted_base_content_ref() { + local base_hash="$1" + local metadata_version="$2" + + printf '%s:base-v%s-%s\n' "${BASE_REPO}" "${metadata_version}" "${base_hash}" +} + +scoped_base_content_ref() { + local base_hash="$1" + local metadata_version="$2" + + printf '%s:base-v%s-preview-%s\n' \ + "${BASE_REPO}" "${metadata_version}" "${base_hash}" +} + +find_matching_base_content_ref() { + local expected_hash="$1" + local expected_version="$2" + local image_ref="" + local digest="" + local immutable_ref="" + local label_values="" + local remote_hash="" + local remote_version="" + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}" + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}" + local attempt=0 + local inspect_status=0 + shift 2 + + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid image identity retry configuration" >&2 + return 1 + fi + + for image_ref in "$@"; do + if ! digest=$(resolve_image_digest "${image_ref}"); then + continue + fi + immutable_ref="${image_ref}@${digest}" + label_values="" + for ((attempt = 1; attempt <= attempts; attempt++)); do + inspect_status=0 + label_values=$(docker buildx imagetools inspect "${immutable_ref}" \ + --format '{{ index .Image.Config.Labels "vllm.rocm_base.content_hash" }} {{ index .Image.Config.Labels "vllm.rocm_base.metadata_version" }}' \ + 2>/dev/null) || inspect_status=$? + remote_hash="" + remote_version="" + if ((inspect_status == 0)) && [[ -n "${label_values}" ]]; then + read -r remote_hash remote_version _ <<< "${label_values}" + if [[ "${remote_hash}" == "${expected_hash}" \ + && "${remote_version}" == "${expected_version}" ]]; then + printf '%s\n' "${immutable_ref}" + return 0 + fi + fi + if ((attempt < attempts)); then + printf \ + 'ROCm base identity lookup incomplete or mismatched for %s (%d/%d); retrying\n' \ + "${immutable_ref}" "${attempt}" "${attempts}" >&2 + sleep "${delay_secs}" + fi + done + if ((inspect_status == 0)) \ + && [[ -n "${remote_hash}" && -n "${remote_version}" ]]; then + echo "ROCm base content ref has unexpected identity: ${immutable_ref}" >&2 + echo " expected hash/version: ${expected_hash} / ${expected_version}" >&2 + echo " found hash/version: ${remote_hash} / ${remote_version}" >&2 + # A content-addressed ref with the wrong identity is corrupt. The + # stable alias is allowed to describe older content and is only a + # migration/recovery candidate. + if [[ "${image_ref}" != "${BASE_REPO}:base" ]]; then + return 2 + fi + fi + echo "Failed to read complete ROCm base identity labels: ${immutable_ref}" >&2 + done + return 1 } rocm_version_from_base_image() { @@ -110,195 +272,51 @@ rocm_version_from_base_image() { tag_component "${version:-${base_image}}" 16 } -git_diff_changed_base() { - local range="$1" - [[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]] -} - -short_git_ref() { - local ref="$1" - - git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}" -} - -extract_arg_default_from_ref() { - local ref="$1" - local arg_name="$2" - local content="" - - content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)" - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ - <<<"${content}" | head -1 -} - -log_arg_default_changes() { - local old_ref="$1" - local new_ref="$2" - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" - local arg_name="" - local old_value="" - local new_value="" - local changed=0 - - echo "Changed ROCm base ARG defaults:" - for arg_name in ${content_args}; do - old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")" - new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")" - if [[ "${old_value}" != "${new_value}" ]]; then - echo " - ${arg_name}: ${old_value:-} -> ${new_value:-}" - changed=1 - fi - done - - if [[ "${changed}" == "0" ]]; then - echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults" - fi -} - -log_arg_line_diff() { - local range="$1" - local arg_diff="" - - arg_diff="$( - git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \ - | awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \ - || true - )" - - if [[ -n "${arg_diff}" ]]; then - echo "Changed Dockerfile ARG lines:" - printf '%s\n' "${arg_diff}" - fi -} - -log_rocm_base_change_check() { - local context="$1" - local range="$2" - local old_ref="$3" - local old_short="" - local head_short="" - - old_short="$(short_git_ref "${old_ref}")" - head_short="$(short_git_ref HEAD)" - - echo "--- :mag: ROCm base refresh check" - echo "Context: ${context}" - echo "Dockerfile: ${DOCKERFILE}" - echo "Base revision: ${old_short}" - echo "Head revision: ${head_short}" - echo "Git diff range: ${range}" -} - -log_rocm_base_rebuild_reason() { - local context="$1" - local range="$2" - local old_ref="$3" - local changed_files="" - - log_rocm_base_change_check "${context}" "${range}" "${old_ref}" - - changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)" - echo "Changed files:" - if [[ -n "${changed_files}" ]]; then - sed 's/^/ - /' <<<"${changed_files}" - else - echo " - ${DOCKERFILE}" - fi - log_arg_default_changes "${old_ref}" HEAD - log_arg_line_diff "${range}" - echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed." -} - -rocm_base_changed_in_range() { - local context="$1" - local range="$2" - local old_ref="$3" - - if git_diff_changed_base "${range}"; then - log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}" - return 0 - fi - - log_rocm_base_change_check "${context}" "${range}" "${old_ref}" - echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged." - return 1 -} - -rocm_base_changed() { - local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" - local base_ref="refs/remotes/origin/${base_branch}" - local merge_base="" - - if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then - echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh" - return 1 - fi - - if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then - echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image" - return 0 - fi - - if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then - echo "Not in a git checkout; skipping ROCm base refresh unless forced" - return 1 - fi - - if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then - git fetch --no-tags --depth=200 origin \ - "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true - merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true) - if [[ -z "${merge_base}" ]]; then - echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced" - return 1 - fi - if rocm_base_changed_in_range \ - "pull request build against ${base_ref}" \ - "${merge_base}...HEAD" \ - "${merge_base}"; then - return 0 - fi - elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \ - && git rev-parse --verify HEAD~1 >/dev/null 2>&1; then - if rocm_base_changed_in_range \ - "stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \ - "HEAD~1..HEAD" \ - "HEAD~1"; then - return 0 - fi - else - git fetch --no-tags --depth=200 origin \ - "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true - merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true) - if [[ -z "${merge_base}" ]]; then - echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced" - return 1 - fi - if rocm_base_changed_in_range \ - "branch build against ${base_ref}" \ - "${merge_base}...HEAD" \ - "${merge_base}"; then - return 0 - fi - fi - - return 1 -} - should_push_stable_tag() { if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then return 1 fi - if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then - return 0 - fi if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then return 1 fi - [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \ - && "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] + is_trusted_main_build +} + +trusted_main_tip_matches_build() { + local branch="${ROCM_BASE_STABLE_BRANCH:-main}" + local build_commit="${BUILDKITE_COMMIT:-}" + local remote_tip="" + + is_trusted_main_build || return 1 + if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Skipping ROCm stable tag: Buildkite commit is missing or invalid" >&2 + return 1 + fi + remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \ + "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }') + if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "Skipping ROCm stable tag: could not resolve remote ${branch} tip" >&2 + return 1 + fi + if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then + echo "Skipping ROCm stable tag: ${branch} changed after ${build_commit} (${remote_tip})" >&2 + return 1 + fi + return 0 +} + +promote_stable_base_tag() { + local source_ref="$1" + local stable_tag="$2" + + ROCM_BASE_STABLE_TAG_UPDATED=0 + should_push_stable_tag && trusted_main_tip_matches_build || return 0 + + docker buildx imagetools create --prefer-index=false \ + -t "${stable_tag}" "${source_ref}" + ROCM_BASE_STABLE_TAG_UPDATED=1 } setup_builder() { @@ -315,29 +333,43 @@ compute_base_content_hash() { local use_sccache="$1" local base_image_digest="$2" local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" + local sccache_arg="" + local sccache_value="" local -a content_paths=() - local -a content_arg_names=() read -r -a content_paths <<< "${content_files}" - read -r -a content_arg_names <<< "${content_args}" { printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")" printf 'dockerfile:%s\n' "${DOCKERFILE}" - printf 'resolved-build-args:\n' - hash_rocm_base_arg_values \ - "${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}" + printf 'base-image-digest:%s\n' "${base_image_digest}" + printf 'use-sccache:%s\n' "${use_sccache}" + if [[ "${use_sccache}" == "1" ]]; then + # These values can change the installed binary or final image + # configuration. SCCACHE_ENDPOINT is transport-only and therefore + # deliberately excluded from image identity. + for sccache_arg in \ + SCCACHE_DOWNLOAD_URL \ + SCCACHE_BUCKET_NAME \ + SCCACHE_REGION_NAME \ + SCCACHE_S3_NO_CREDENTIALS; do + sccache_value="${!sccache_arg:-}" + if [[ -z "${sccache_value}" ]]; then + sccache_value=$(extract_arg_default "${sccache_arg}") + fi + printf 'arg:%s=%s\n' \ + "${sccache_arg}" "${sccache_value:-}" + done + fi } | sha256sum | cut -d' ' -f1 } build_base_image() { local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}" local base_hash="" - local build_date="" - local build_suffix="" local base_image_arg="" local base_image_digest="" + local pinned_base_image="" local rocm_version="" local triton_arg="" local pytorch_arg="" @@ -348,20 +380,19 @@ build_base_image() { local mori_arg="" local python_version_arg="" local pytorch_rocm_arch_arg="" - local pytorch_branch="" - local aiter_branch="" local dependency_summary="" - local descriptor="" - local ci_descriptor="" - local descriptive_tag="" local stable_tag="${BASE_REPO}:base" - local ci_descriptive_tag="" + local trusted_content_tag="" + local scoped_content_tag="" + local writable_content_tag="" + local build_ref="" + local immutable_ref="" + local selected_digest="" + local stable_digest="" + local reuse_status=0 local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" local content_files_hash="" local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}" - local -a tags=() - local -a no_cache_args=() local -a sccache_args=() local -a content_paths=() @@ -369,13 +400,21 @@ build_base_image() { echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2 exit 1 fi - - build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}" - if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then - build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}" + if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then + echo "Invalid ROCm base metadata version: ${metadata_version}" >&2 + return 1 fi + if [[ "${use_sccache}" != "0" && "${use_sccache}" != "1" ]]; then + echo "ROCm base USE_SCCACHE must be 0 or 1: ${use_sccache}" >&2 + return 1 + fi + base_image_arg="$(extract_arg_default BASE_IMAGE)" - base_image_digest="$(resolve_image_digest "${base_image_arg}")" + if ! base_image_digest=$(resolve_image_digest "${base_image_arg}"); then + echo "Failed to resolve ROCm base input image: ${base_image_arg}" >&2 + return 1 + fi + pinned_base_image="${base_image_arg%@*}@${base_image_digest}" read -r -a content_paths <<< "${content_files}" content_files_hash="$(compute_content_hash "${content_paths[@]}")" base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}") @@ -389,125 +428,161 @@ build_base_image() { mori_arg="$(extract_arg_default MORI_BRANCH)" python_version_arg="$(extract_arg_default PYTHON_VERSION)" pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)" - pytorch_branch="$(tag_component "${pytorch_arg}" 16)" - aiter_branch="$(tag_component "${aiter_arg}" 24)" dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}" - descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")" - ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")" - - descriptive_tag="${BASE_REPO}:${descriptor}" - ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}" - - tags=(-t "${descriptive_tag}") - if should_push_stable_tag; then - tags+=(-t "${stable_tag}") - metadata_set "rocm-base-push-stable-tag" "1" + trusted_content_tag=$(trusted_base_content_ref "${base_hash}" "${metadata_version}") + scoped_content_tag=$(scoped_base_content_ref "${base_hash}" "${metadata_version}") + # Preview writes share an exact-content namespace so stacked PRs can reuse + # identical images. They may import the trusted ref, but never overwrite it. + if is_trusted_main_build; then + scoped_content_tag="${trusted_content_tag}" + writable_content_tag="${trusted_content_tag}" else - metadata_set "rocm-base-push-stable-tag" "0" + writable_content_tag="${scoped_content_tag}" fi - if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then - no_cache_args=(--no-cache) + configure_rocm_base_layer_cache + + if [[ "${use_sccache}" == "1" ]]; then + for env_name in \ + SCCACHE_DOWNLOAD_URL \ + SCCACHE_ENDPOINT \ + SCCACHE_BUCKET_NAME \ + SCCACHE_REGION_NAME \ + SCCACHE_S3_NO_CREDENTIALS; do + if [[ -n "${!env_name:-}" ]]; then + sccache_args+=(--build-arg "${env_name}=${!env_name}") + fi + done fi - for env_name in \ - SCCACHE_DOWNLOAD_URL \ - SCCACHE_ENDPOINT \ - SCCACHE_BUCKET_NAME \ - SCCACHE_REGION_NAME \ - SCCACHE_S3_NO_CREDENTIALS; do - if [[ -n "${!env_name:-}" ]]; then - sccache_args+=(--build-arg "${env_name}=${!env_name}") - fi - done - - echo "--- :docker: Building ROCm base image" + echo "--- :docker: Preparing ROCm base image" echo "Dockerfile: ${DOCKERFILE}" - echo "Descriptive tag: ${descriptive_tag}" + echo "Trusted content tag: ${trusted_content_tag}" + echo "Writable content tag: ${writable_content_tag}" echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))" echo "Content hash: ${base_hash}" echo "Dependency summary: ${dependency_summary}" echo "USE_SCCACHE: ${use_sccache}" + echo "BuildKit layer cache: ${ROCM_BASE_LAYER_CACHE_REF}" - docker buildx build \ - "${no_cache_args[@]}" \ - --pull \ - --progress "${BUILDKIT_PROGRESS:-plain}" \ - --file "${DOCKERFILE}" \ - --build-arg "USE_SCCACHE=${use_sccache}" \ - "${sccache_args[@]}" \ - --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \ - --label "org.opencontainers.image.vendor=vLLM" \ - --label "org.opencontainers.image.title=vLLM ROCm base" \ - --label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \ - --label "vllm.rocm_base.metadata_version=${metadata_version}" \ - --label "vllm.rocm_base.content_hash=${base_hash}" \ - --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \ - --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \ - --label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \ - --label "vllm.rocm_base.image.stable=${stable_tag}" \ - --label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \ - --label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \ - --label "vllm.rocm_base.descriptor=${descriptor}" \ - --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \ - --label "vllm.rocm_base.base_image=${base_image_arg}" \ - --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \ - --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \ - --label "vllm.rocm_base.dependency.python=${python_version_arg}" \ - --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \ - --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \ - --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \ - --label "vllm.rocm_base.dependency.triton=${triton_arg}" \ - --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \ - --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \ - --label "vllm.rocm_base.dependency.mori=${mori_arg}" \ - --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \ - "${tags[@]}" \ - --push \ - . + if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" != "1" \ + && "${ROCM_BASE_NO_CACHE:-0}" != "1" ]]; then + reuse_status=0 + if [[ "${trusted_content_tag}" == "${scoped_content_tag}" ]]; then + immutable_ref=$(find_matching_base_content_ref \ + "${base_hash}" "${metadata_version}" \ + "${trusted_content_tag}" "${stable_tag}") || reuse_status=$? + else + immutable_ref=$(find_matching_base_content_ref \ + "${base_hash}" "${metadata_version}" \ + "${trusted_content_tag}" "${scoped_content_tag}" \ + "${stable_tag}") \ + || reuse_status=$? + fi + else + reuse_status=1 + echo "Forced/no-cache ROCm base build; bypassing content image reuse" + fi - docker buildx imagetools inspect "${descriptive_tag}" >/dev/null + if ((reuse_status > 1)); then + return "${reuse_status}" + fi + if [[ ${reuse_status} -eq 0 && -n "${immutable_ref}" ]]; then + echo "Reusing ROCm base image with matching content: ${immutable_ref}" + if [[ "${immutable_ref%@*}" == "${stable_tag}" \ + && "${writable_content_tag}" != "${stable_tag}" ]]; then + echo "Backfilling ROCm base content tag: ${writable_content_tag}" + docker buildx imagetools create --prefer-index=false \ + -t "${writable_content_tag}" "${immutable_ref}" + if ! immutable_ref=$(find_matching_base_content_ref \ + "${base_hash}" "${metadata_version}" \ + "${writable_content_tag}"); then + echo "Backfilled ROCm base content tag failed identity validation" >&2 + return 1 + fi + fi + else + build_ref="${writable_content_tag}" + echo "No reusable ROCm base content image found; building ${build_ref}" + setup_builder + docker buildx build \ + "${ROCM_BASE_CACHE_ARGS[@]}" \ + --pull \ + --provenance=false \ + --progress "${BUILDKIT_PROGRESS:-plain}" \ + --file "${DOCKERFILE}" \ + --build-arg "BASE_IMAGE=${pinned_base_image}" \ + --build-arg "USE_SCCACHE=${use_sccache}" \ + "${sccache_args[@]}" \ + --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \ + --label "org.opencontainers.image.vendor=vLLM" \ + --label "org.opencontainers.image.title=vLLM ROCm base" \ + --label "vllm.rocm_base.metadata_version=${metadata_version}" \ + --label "vllm.rocm_base.content_hash=${base_hash}" \ + --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \ + --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \ + --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \ + --label "vllm.rocm_base.base_image=${pinned_base_image}" \ + --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \ + --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \ + --label "vllm.rocm_base.dependency.python=${python_version_arg}" \ + --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \ + --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \ + --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \ + --label "vllm.rocm_base.dependency.triton=${triton_arg}" \ + --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \ + --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \ + --label "vllm.rocm_base.dependency.mori=${mori_arg}" \ + --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \ + -t "${build_ref}" \ + --push \ + . + if ! immutable_ref=$(find_matching_base_content_ref \ + "${base_hash}" "${metadata_version}" "${build_ref}"); then + echo "Published ROCm base image failed identity validation: ${build_ref}" >&2 + return 1 + fi + fi + + selected_digest="${immutable_ref##*@}" + metadata_set "rocm-base-image" "${immutable_ref}" + metadata_set "rocm-base-push-stable-tag" "0" + stable_digest=$(resolve_image_digest "${stable_tag}" 2>/dev/null || true) + if [[ -n "${stable_digest}" && "${stable_digest}" == "${selected_digest}" ]]; then + echo "--- :white_check_mark: Existing ROCm base is current" + echo "Stable ${stable_tag} already resolves to ${selected_digest}" + return 0 + fi + + promote_stable_base_tag "${immutable_ref}" "${stable_tag}" + + metadata_set "rocm-base-push-stable-tag" "${ROCM_BASE_STABLE_TAG_UPDATED}" metadata_set "rocm-base-refresh" "1" - metadata_set "rocm-base-image" "${descriptive_tag}" - metadata_set "rocm-base-image-descriptive" "${descriptive_tag}" - metadata_set "rocm-base-image-stable" "${stable_tag}" - metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}" - metadata_set "rocm-base-metadata-version" "${metadata_version}" - metadata_set "rocm-base-content-hash" "${base_hash}" - metadata_set "rocm-base-content-files-hash" "${content_files_hash}" - metadata_set "rocm-base-content-files" "${content_files}" - metadata_set "rocm-base-content-args" "${content_args}" - metadata_set "rocm-base-base-image-digest" "${base_image_digest}" - metadata_set "rocm-base-dockerfile" "${DOCKERFILE}" - metadata_set "rocm-base-descriptor" "${descriptor}" - metadata_set "rocm-base-dependency-summary" "${dependency_summary}" - metadata_set "rocm-base-dependency-rocm" "${rocm_version}" - metadata_set "rocm-base-dependency-python" "${python_version_arg}" - metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}" - metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}" - metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}" - metadata_set "rocm-base-dependency-triton" "${triton_arg}" - metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}" - metadata_set "rocm-base-dependency-aiter" "${aiter_arg}" - metadata_set "rocm-base-dependency-mori" "${mori_arg}" - metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}" - metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}" echo "--- :white_check_mark: ROCm base image published" - echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds" + echo "Use BASE_IMAGE=${immutable_ref} for downstream ROCm CI builds" } main() { metadata_set "rocm-base-refresh" "0" - if ! rocm_base_changed; then - echo "ROCm base Dockerfile did not change; skipping base image refresh" + if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then + local stable_ref="${BASE_REPO}:base" + local stable_digest="" + + if ! stable_digest=$(resolve_image_digest "${stable_ref}"); then + echo "Could not pin explicitly skipped ROCm base: ${stable_ref}" >&2 + return 1 + fi + metadata_set "rocm-base-image" "${stable_ref}@${stable_digest}" + metadata_set "rocm-base-push-stable-tag" "0" + echo "ROCM_BASE_REFRESH_SKIP=1; pinned current stable base at ${stable_digest}" return 0 fi - - setup_builder build_base_image } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/.buildkite/scripts/rocm/smoke-test-image.sh b/.buildkite/scripts/rocm/smoke-test-image.sh index ed511c9b77..20f5f5623c 100755 --- a/.buildkite/scripts/rocm/smoke-test-image.sh +++ b/.buildkite/scripts/rocm/smoke-test-image.sh @@ -3,7 +3,18 @@ set -euo pipefail -image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}" +if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then + base_refreshed="" + if command -v buildkite-agent >/dev/null 2>&1; then + base_refreshed="$(buildkite-agent meta-data get rocm-base-refresh 2>/dev/null || true)" + fi + if [[ "${base_refreshed}" != "1" ]]; then + echo "ROCM_CI_ARTIFACT_ONLY=1; no full image was built, skipping smoke test" + exit 0 + fi +fi + +image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}" docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec ' if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi diff --git a/.dockerignore b/.dockerignore index 66447272e9..e0284129ee 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,8 @@ /.venv /build +/artifacts/vllm-rocm-install/ +/bake-config-build-*.json +/wheel-export/ dist vllm/*.so vllm/vllm-rs diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index d6ddb5d173..f612a0bd0e 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -253,21 +253,32 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry, sccache --show-stats; \ fi +# ----------------------- +# Shared vLLM build dependencies +# +# Install requirements once, before bringing source-specific inputs into either +# consumer. Native compilation and wheel packaging then share this expensive +# layer while retaining their independent source cache boundaries. +FROM base AS build_vllm_dependencies +ARG COMMON_WORKDIR +ENV VLLM_TARGET_DEVICE=rocm +WORKDIR ${COMMON_WORKDIR}/vllm + +COPY --from=fetch_vllm \ + ${COMMON_WORKDIR}/vllm/requirements/common.txt \ + ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \ + requirements/ + +RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ + uv pip install --system -r requirements/rocm.txt + # ----------------------- # vLLM native build stages # # csrc-build intentionally copies only files that affect ROCm native extension # compilation. That keeps unrelated CI/test/docs edits from invalidating the # expensive HIP/C++ build layer. -FROM base AS csrc-build -ARG COMMON_WORKDIR -WORKDIR ${COMMON_WORKDIR}/vllm - -COPY requirements/rocm.txt requirements/rocm.txt -COPY requirements/common.txt requirements/common.txt -RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ - uv pip install --system -r requirements/rocm.txt - +FROM build_vllm_dependencies AS csrc-build # pyproject.toml is bind-mounted so it is not persisted in the csrc-build image, # but it still participates in the BuildKit cache key for this RUN step. COPY setup.py CMakeLists.txt ./ @@ -277,7 +288,6 @@ COPY csrc csrc/ COPY vllm/envs.py vllm/envs.py COPY vllm/__init__.py vllm/__init__.py -ENV VLLM_TARGET_DEVICE=rocm ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build" RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \ @@ -296,11 +306,10 @@ RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproj # Build the full vLLM ROCm wheel by reusing the native extension wheel from # csrc-build. This stage still rebuilds for Python/package changes, but skips -# the expensive HIP/C++ compile when native inputs are unchanged. -FROM fetch_vllm AS build_vllm -ARG COMMON_WORKDIR -ENV VLLM_TARGET_DEVICE=rocm - +# the expensive dependency installation and HIP/C++ compilation when their +# respective inputs are unchanged. +FROM build_vllm_dependencies AS build_vllm +COPY --from=fetch_vllm ${COMMON_WORKDIR}/vllm ${COMMON_WORKDIR}/vllm COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels # Drop the pre-built Rust artifacts into the source tree. setup.py detects @@ -308,9 +317,7 @@ COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/ -RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ - cd vllm \ - && uv pip install --system -r requirements/rocm.txt \ +RUN cd "${COMMON_WORKDIR}/vllm" \ && export VLLM_USE_PRECOMPILED=1 \ && export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \ && export VLLM_DOCKER_BUILD_CONTEXT=1 \ diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 0bba4be5bb..4d5575cefd 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -134,11 +134,7 @@ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install ### FROM base AS build_pytorch ARG PYTORCH_BRANCH -ARG PYTORCH_VISION_BRANCH -ARG PYTORCH_AUDIO_BRANCH ARG PYTORCH_REPO -ARG PYTORCH_VISION_REPO -ARG PYTORCH_AUDIO_REPO ARG USE_SCCACHE RUN apt-get update && apt-get install -y pkg-config liblzma-dev @@ -156,8 +152,27 @@ RUN cd pytorch && python3 tools/amd_build/build_amd.py \ && sccache --show-stats; \ fi \ && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ - && pip install dist/*.whl + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install + + +### +### Shared PyTorch runtime for dependent builds +### +FROM base AS build_pytorch_runtime +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13 +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \ + pip install /install/*.whl + + +### +### TorchVision build +### +FROM build_pytorch_runtime AS build_torchvision +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_VISION_REPO +ARG USE_SCCACHE + RUN git clone ${PYTORCH_VISION_REPO} vision RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \ && if [ "$USE_SCCACHE" = "1" ]; then \ @@ -166,23 +181,31 @@ RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \ && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ fi \ && python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ - && pip install dist/*.whl + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install + + +### +### TorchAudio build +### +FROM build_pytorch_runtime AS build_torchaudio +ARG PYTORCH_AUDIO_BRANCH +ARG PYTORCH_AUDIO_REPO +ARG USE_SCCACHE + RUN git clone ${PYTORCH_AUDIO_REPO} audio RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \ - && git submodule update --init --recursive \ - && pip install -r requirements.txt \ + && git submodule update --init --recursive +RUN cd audio && pip install -r requirements.txt +RUN cd audio \ && if [ "$USE_SCCACHE" = "1" ]; then \ export HIP_CLANG_PATH=/opt/sccache-wrappers \ && export CMAKE_C_COMPILER_LAUNCHER=sccache \ && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ fi \ && python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ - && pip install dist/*.whl -RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \ - && cp /app/vision/dist/*.whl /app/install \ - && cp /app/audio/dist/*.whl /app/install + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install ### @@ -263,6 +286,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ cp /install/*.whl /app/debs @@ -277,6 +304,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ cp /install/*.whl /app/debs RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ @@ -315,4 +346,4 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ - && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \ No newline at end of file + && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt diff --git a/docker/ci-rocm.hcl b/docker/ci-rocm.hcl index 7062a0be93..0bdf4d1a49 100644 --- a/docker/ci-rocm.hcl +++ b/docker/ci-rocm.hcl @@ -338,33 +338,22 @@ group "test-rocm-ci-with-artifacts" { targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"] } -# Full test image + wheel export. Kept for fallback/debugging when a pushed -# per-commit image is useful. +# Full test image + wheel export. Kept for fallback/debugging when a pushed, +# build-scoped image is useful. group "test-rocm-ci-with-wheel" { targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"] } -# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG -# to the primary tag for this build. Builds always publish a content-scoped tag -# when the ci_base content hash is available. Builds with BUILDKITE_COMMIT also -# publish a commit-scoped tag, either as the primary tag or an additional alias. -# NIGHTLY=1 builds on the stable branch can additionally set -# CI_BASE_IMAGE_TAG_STABLE to refresh rocm/vllm-dev:ci_base. +# Primary output tag for the ci_base build. In Buildkite this is a unique, +# build-scoped tag; ci-bake-rocm.sh validates it before creating content and +# stable aliases. variable "CI_BASE_IMAGE_TAG" { default = "rocm/vllm-dev:ci_base" } -# Supplemental tags only. ci-bake-rocm.sh leaves these empty when the same ref -# is already the primary CI_BASE_IMAGE_TAG. -variable "CI_BASE_IMAGE_TAG_COMMIT_EXTRA" { - default = "" -} - -variable "CI_BASE_IMAGE_TAG_CONTENT_EXTRA" { - default = "" -} - -variable "CI_BASE_IMAGE_TAG_STABLE" { +# Versioned, content-addressed trusted ref. Preview builds import it read-only; +# the wrapper creates their source-scoped content alias after validation. +variable "CI_BASE_TRUSTED_CONTENT_REF" { default = "" } @@ -403,21 +392,23 @@ target "deepep-rocm-ci" { # Uses inline cache metadata on the ci_base image itself instead of exporting a # separate registry cache artifact. target "ci-base-rocm-ci" { - inherits = ["_common-rocm", "_ci-rocm", "_labels"] + inherits = ["_common-rocm"] target = "ci_base" + args = { + max_jobs = CI_MAX_JOBS + } cache-from = concat( compact([ CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "", - CI_BASE_IMAGE_TAG_COMMIT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_COMMIT_EXTRA}" : "", - CI_BASE_IMAGE_TAG_CONTENT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_CONTENT_EXTRA}" : "", - CI_BASE_IMAGE_TAG_STABLE != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_STABLE}" : "", + CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "", ]), # Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages # are cache hits even when ci_base itself needs rebuilding. get_cache_from_rocm_deps(), ) cache-to = ["type=inline"] - tags = compact([CI_BASE_IMAGE_TAG, CI_BASE_IMAGE_TAG_COMMIT_EXTRA, CI_BASE_IMAGE_TAG_CONTENT_EXTRA, CI_BASE_IMAGE_TAG_STABLE]) + tags = compact([CI_BASE_IMAGE_TAG]) + attest = ["type=provenance,disabled=true"] output = ["type=registry"] } diff --git a/tests/tools/test_docker_build_metadata_args.py b/tests/tools/test_docker_build_metadata_args.py index 72d4ad8089..925ed4bfc9 100644 --- a/tests/tools/test_docker_build_metadata_args.py +++ b/tests/tools/test_docker_build_metadata_args.py @@ -171,10 +171,6 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None: "requirements/common.txt", "requirements/rocm.txt", "requirements/test/rocm.txt", - "docker/Dockerfile.rocm_base", "docker/Dockerfile.rocm", - "docker/ci-rocm.hcl", - "docker/docker-bake-rocm.hcl", - ".buildkite/scripts/ci-bake-rocm.sh", ): assert expected in ci_bake