From cbb5f045beb1f82bc139e14a4d3b2a904a8b358e Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 7 Jul 2026 05:10:50 -0500 Subject: [PATCH] [ROCm][CI] Refresh ROCm base images when docker rocm_base changes (#46904) Signed-off-by: Rohan138 Signed-off-by: Codex Signed-off-by: Micah Williamson Signed-off-by: Matthew Wong Signed-off-by: Andreas Karatzas Co-authored-by: Rohan138 Co-authored-by: Codex Co-authored-by: Micah Williamson Co-authored-by: Matthew Wong --- .buildkite/ci_config_rocm.yaml | 1 + .buildkite/hardware_tests/amd.yaml | 61 ++- .buildkite/scripts/ci-bake-rocm.sh | 28 +- .../scripts/hardware_ci/run-amd-test.sh | 26 +- .buildkite/scripts/rocm/build-ci-base.sh | 32 ++ .buildkite/scripts/rocm/build-test-image.sh | 57 ++ .buildkite/scripts/rocm/refresh-base-image.sh | 513 ++++++++++++++++++ .buildkite/scripts/rocm/smoke-test-image.sh | 32 ++ docker/Dockerfile.rocm_base | 2 +- docs/design/attention_backends.md | 2 +- tests/compile/passes/test_fusion_attn.py | 5 - .../attention/test_rocm_aiter_unified_attn.py | 3 +- .../test_rocm_attention_backends_selection.py | 10 +- .../backends/rocm_aiter_unified_attn.py | 4 +- 14 files changed, 724 insertions(+), 52 deletions(-) create mode 100755 .buildkite/scripts/rocm/build-ci-base.sh create mode 100755 .buildkite/scripts/rocm/build-test-image.sh create mode 100755 .buildkite/scripts/rocm/refresh-base-image.sh create mode 100755 .buildkite/scripts/rocm/smoke-test-image.sh diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml index 23f3234007..408ccee58e 100644 --- a/.buildkite/ci_config_rocm.yaml +++ b/.buildkite/ci_config_rocm.yaml @@ -8,6 +8,7 @@ run_all_patterns: - "docker/docker-bake-rocm.hcl" - ".buildkite/hardware_tests/amd.yaml" - ".buildkite/scripts/ci-bake-rocm.sh" + - ".buildkite/scripts/rocm/" - ".buildkite/scripts/hardware_ci/run-amd-test.py" - ".buildkite/scripts/hardware_ci/run-amd-test.sh" - "CMakeLists.txt" diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml index a18241cf18..d47a7e394a 100644 --- a/.buildkite/hardware_tests/amd.yaml +++ b/.buildkite/hardware_tests/amd.yaml @@ -1,5 +1,28 @@ group: Hardware - AMD Build + +# ROCm image flow: +# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes. +# 2. Build ci_base from either the stable base or the freshly refreshed base. +# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run. steps: + - label: "AMD: :docker: refresh ROCm base" + key: refresh-rocm-base-amd + depends_on: [] + device: amd_cpu + no_plugin: true + commands: + - bash .buildkite/scripts/rocm/refresh-base-image.sh + env: + DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: "tty" + TERM: "xterm-256color" + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 1 + - exit_status: -10 # Agent was lost + limit: 1 + # Ensure ci_base is up-to-date before building the test image. # Compares a content hash of ci_base-affecting files against the remote # image label. If hashes match the build is skipped (< 30 s); if they @@ -7,13 +30,16 @@ steps: - label: "AMD: :docker: ensure ci_base" key: ensure-ci-base-amd soft_fail: false - depends_on: [] + depends_on: + - refresh-rocm-base-amd device: amd_cpu no_plugin: true commands: - - bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps + - bash .buildkite/scripts/rocm/build-ci-base.sh env: DOCKER_BUILDKIT: "1" + BUILDKIT_PROGRESS: "tty" + TERM: "xterm-256color" VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" REMOTE_VLLM: "1" @@ -33,35 +59,12 @@ steps: device: amd_cpu no_plugin: true commands: - - | - if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then - echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only" - IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts - else - bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel - fi - - | - docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec ' - if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi - if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi - if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi - if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi - command -v python3 - command -v uv - command -v pytest - if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then - echo No ROCm CLI found in image >&2 - exit 1 - fi - python3 - </dev/null 2>&1; then + buildkite-agent meta-data set "${key}" "${value}" || true + fi +} + get_remote_image_label() { local image_ref="$1" local label_key="$2" @@ -733,6 +743,8 @@ configure_ci_base_image_refs() { if is_ci_base_target; then IMAGE_TAG="${primary_tag}" + CI_BASE_IMAGE="${primary_tag}" + export CI_BASE_IMAGE export IMAGE_TAG echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}" @@ -750,6 +762,10 @@ configure_ci_base_image_refs() { echo "ci_base stable alias will not be pushed for this build" echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}" fi + set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}" + set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}" + set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT:-}" + set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}" return 0 fi @@ -1779,7 +1795,10 @@ seed_dependency_caches_if_needed() { echo "--- :docker: Seeding ${target}" echo "Expected cache ref: ${cache_ref}" - docker buildx bake "${BAKE_FILES[@]}" --progress plain "${target}" + docker buildx bake \ + "${BAKE_FILES[@]}" \ + --progress "${BUILDKIT_PROGRESS:-plain}" \ + "${target}" verify_dependency_cache_ref "${cache_ref}" done } @@ -1807,7 +1826,10 @@ run_bake() { local build_rc=0 echo "--- :docker: Building ${TARGET}" - docker buildx bake "${BAKE_FILES[@]}" --progress plain "${BAKE_TARGETS[@]}" || build_rc=$? + docker buildx bake \ + "${BAKE_FILES[@]}" \ + --progress "${BUILDKIT_PROGRESS:-plain}" \ + "${BAKE_TARGETS[@]}" || build_rc=$? if [[ ${build_rc} -eq 0 ]]; then echo "--- :white_check_mark: Build complete" diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 3969c7f974..fee9ab04f4 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -28,6 +28,17 @@ ############################################################################### set -o pipefail +: "${BUILDKIT_PROGRESS:=plain}" +: "${TERM:=xterm-256color}" +: "${FORCE_COLOR:=1}" +: "${CLICOLOR_FORCE:=1}" +: "${PY_COLORS:=1}" +: "${ROCM_DOCKER_TTY:=1}" +if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" +fi +export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS ROCM_DOCKER_TTY + # Export Python path for commands that run directly on the host. Containerized # tests set this to /vllm-workspace below so spawned Python processes do not # depend on their current working directory. @@ -149,6 +160,7 @@ EOF echo "--- Building local ROCm test image" docker build \ --pull=false \ + --progress "${BUILDKIT_PROGRESS}" \ --build-arg "BASE_IMAGE=${base_image}" \ -t "${artifact_image}" \ "${context_dir}" || return 1 @@ -535,6 +547,13 @@ if is_multi_node "$commands"; then else echo "--- Single-node job" echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" + docker_run_terminal_args=(-i) + if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then + docker_run_terminal_args+=(-t) + echo "Docker interactive stdin: enabled; TTY allocation: enabled" + else + echo "Docker interactive stdin: enabled; TTY allocation: disabled" + fi ulimit_core_hard=$(ulimit -H -c) if [[ "$ulimit_core_hard" == "unlimited" ]]; then @@ -551,7 +570,7 @@ else fi docker run \ - -t -i \ + "${docker_run_terminal_args[@]}" \ --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ $RDMA_FLAGS \ --network=host \ @@ -566,6 +585,11 @@ else -e AWS_SECRET_ACCESS_KEY \ -e BUILDKITE_PARALLEL_JOB \ -e BUILDKITE_PARALLEL_JOB_COUNT \ + -e TERM \ + -e FORCE_COLOR \ + -e CLICOLOR_FORCE \ + -e PY_COLORS \ + -e PYTEST_ADDOPTS \ -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ diff --git a/.buildkite/scripts/rocm/build-ci-base.sh b/.buildkite/scripts/rocm/build-ci-base.sh new file mode 100755 index 0000000000..23d17e17b4 --- /dev/null +++ b/.buildkite/scripts/rocm/build-ci-base.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build the ROCm ci_base image, optionally from a freshly rebuilt ROCm base. + +set -euo pipefail + +metadata_get() { + local key="$1" + if command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent meta-data get "${key}" 2>/dev/null || true + fi +} + +main() { + local base_refreshed="" + + base_refreshed="$(metadata_get rocm-base-refresh)" + if [[ "${base_refreshed}" == "1" ]]; then + export BASE_IMAGE + export CI_BASE_PUSH_STABLE_TAG + + BASE_IMAGE="$(metadata_get rocm-base-image)" + CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)" + CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}" + + echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}" + echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}" + fi + + bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps +} + +main "$@" diff --git a/.buildkite/scripts/rocm/build-test-image.sh b/.buildkite/scripts/rocm/build-test-image.sh new file mode 100755 index 0000000000..9803e20d02 --- /dev/null +++ b/.buildkite/scripts/rocm/build-test-image.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Build the ROCm CI test image or wheel artifact. +# +# When Dockerfile.rocm_base changes, always build the full image so downstream +# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain. + +set -euo pipefail + +metadata_get() { + local key="$1" + if command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent meta-data get "${key}" 2>/dev/null || true + fi +} + +use_refreshed_base_if_present() { + local base_refreshed="" + + base_refreshed="$(metadata_get rocm-base-refresh)" + if [[ "${base_refreshed}" != "1" ]]; then + return 1 + fi + + export BASE_IMAGE + export CI_BASE_IMAGE + export IMAGE_TAG_LATEST + + BASE_IMAGE="$(metadata_get rocm-base-image)" + CI_BASE_IMAGE="$(metadata_get rocm-ci-base-image)" + IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)" + + echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}" + echo "Using refreshed ROCm ci_base image for test image: ${CI_BASE_IMAGE}" + if [[ -n "${IMAGE_TAG_LATEST}" ]]; then + echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}" + fi + + return 0 +} + +main() { + local base_refreshed=0 + + if use_refreshed_base_if_present; then + base_refreshed=1 + fi + + if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" && "${base_refreshed}" != "1" ]]; then + echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only" + IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts + return + fi + + bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel +} + +main "$@" diff --git a/.buildkite/scripts/rocm/refresh-base-image.sh b/.buildkite/scripts/rocm/refresh-base-image.sh new file mode 100755 index 0000000000..06e3e80c96 --- /dev/null +++ b/.buildkite/scripts/rocm/refresh-base-image.sh @@ -0,0 +1,513 @@ +#!/usr/bin/env bash +# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes. +# +# Normal AMD CI builds should not pay for this path. The script no-ops unless +# docker/Dockerfile.rocm_base changed relative to the branch base, the previous +# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set. + +set -euo pipefail + +DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}" +BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}" +CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}" +BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}" +DEFAULT_ROCM_BASE_METADATA_VERSION="1" +DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}" +DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE" + +metadata_set() { + local key="$1" + local value="$2" + + [[ -n "${value}" ]] || return 0 + if command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent meta-data set "${key}" "${value}" || true + fi +} + +compute_content_hash() { + local path="" + local file="" + + for path in "$@"; do + if [[ -d "${path}" ]]; then + while IFS= read -r -d '' file; do + printf 'file:%s\n' "${file}" + sha256sum "${file}" + done < <(find "${path}" -type f -print0 | sort -z) + elif [[ -f "${path}" ]]; then + printf 'file:%s\n' "${path}" + sha256sum "${path}" + else + printf 'missing:%s\n' "${path}" + fi + done | sha256sum | cut -d' ' -f1 +} + +clean_docker_tag() { + local input="$1" + echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 +} + +tag_component() { + local input="$1" + local max_chars="${2:-24}" + + clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}" +} + +extract_arg_default() { + local arg_name="$1" + + sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ + "${DOCKERFILE}" | head -1 +} + +resolve_image_digest() { + local image_ref="$1" + + docker buildx imagetools inspect "${image_ref}" 2>/dev/null \ + | sed -n -E 's/^Digest:[[:space:]]+//p' \ + | head -1 || true +} + +resolve_rocm_base_arg_value() { + local arg_name="$1" + local use_sccache="$2" + + case "${arg_name}" in + USE_SCCACHE) + printf '%s\n' "${use_sccache}" + ;; + *) + extract_arg_default "${arg_name}" + ;; + esac +} + +hash_rocm_base_arg_values() { + local use_sccache="$1" + local base_image_digest="$2" + local arg_name="" + local arg_value="" + shift 2 || true + + for arg_name in "$@"; do + [[ -n "${arg_name}" ]] || continue + arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}") + printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-}" + if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then + printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}" + fi + done +} + +rocm_version_from_base_image() { + local base_image="$1" + local version="" + + version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")" + tag_component "${version:-${base_image}}" 16 +} + +git_diff_changed_base() { + local range="$1" + [[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]] +} + +short_git_ref() { + local ref="$1" + + git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}" +} + +extract_arg_default_from_ref() { + local ref="$1" + local arg_name="$2" + local content="" + + content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)" + sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ + <<<"${content}" | head -1 +} + +log_arg_default_changes() { + local old_ref="$1" + local new_ref="$2" + local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" + local arg_name="" + local old_value="" + local new_value="" + local changed=0 + + echo "Changed ROCm base ARG defaults:" + for arg_name in ${content_args}; do + old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")" + new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")" + if [[ "${old_value}" != "${new_value}" ]]; then + echo " - ${arg_name}: ${old_value:-} -> ${new_value:-}" + changed=1 + fi + done + + if [[ "${changed}" == "0" ]]; then + echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults" + fi +} + +log_arg_line_diff() { + local range="$1" + local arg_diff="" + + arg_diff="$( + git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \ + | awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \ + || true + )" + + if [[ -n "${arg_diff}" ]]; then + echo "Changed Dockerfile ARG lines:" + printf '%s\n' "${arg_diff}" + fi +} + +log_rocm_base_change_check() { + local context="$1" + local range="$2" + local old_ref="$3" + local old_short="" + local head_short="" + + old_short="$(short_git_ref "${old_ref}")" + head_short="$(short_git_ref HEAD)" + + echo "--- :mag: ROCm base refresh check" + echo "Context: ${context}" + echo "Dockerfile: ${DOCKERFILE}" + echo "Base revision: ${old_short}" + echo "Head revision: ${head_short}" + echo "Git diff range: ${range}" +} + +log_rocm_base_rebuild_reason() { + local context="$1" + local range="$2" + local old_ref="$3" + local changed_files="" + + log_rocm_base_change_check "${context}" "${range}" "${old_ref}" + + changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)" + echo "Changed files:" + if [[ -n "${changed_files}" ]]; then + sed 's/^/ - /' <<<"${changed_files}" + else + echo " - ${DOCKERFILE}" + fi + log_arg_default_changes "${old_ref}" HEAD + log_arg_line_diff "${range}" + echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed." +} + +rocm_base_changed_in_range() { + local context="$1" + local range="$2" + local old_ref="$3" + + if git_diff_changed_base "${range}"; then + log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}" + return 0 + fi + + log_rocm_base_change_check "${context}" "${range}" "${old_ref}" + echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged." + return 1 +} + +rocm_base_changed() { + local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + local base_ref="refs/remotes/origin/${base_branch}" + local merge_base="" + + if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then + echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh" + return 1 + fi + + if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then + echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image" + return 0 + fi + + if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "Not in a git checkout; skipping ROCm base refresh unless forced" + return 1 + fi + + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + git fetch --no-tags --depth=200 origin \ + "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true + merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true) + if [[ -z "${merge_base}" ]]; then + echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced" + return 1 + fi + if rocm_base_changed_in_range \ + "pull request build against ${base_ref}" \ + "${merge_base}...HEAD" \ + "${merge_base}"; then + return 0 + fi + elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \ + && git rev-parse --verify HEAD~1 >/dev/null 2>&1; then + if rocm_base_changed_in_range \ + "stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \ + "HEAD~1..HEAD" \ + "HEAD~1"; then + return 0 + fi + else + git fetch --no-tags --depth=200 origin \ + "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true + merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true) + if [[ -z "${merge_base}" ]]; then + echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced" + return 1 + fi + if rocm_base_changed_in_range \ + "branch build against ${base_ref}" \ + "${merge_base}...HEAD" \ + "${merge_base}"; then + return 0 + fi + fi + + return 1 +} + +should_push_stable_tag() { + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then + return 1 + fi + + if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then + return 0 + fi + if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then + return 1 + fi + + [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \ + && "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] +} + +setup_builder() { + echo "--- :buildkite: Setting up buildx builder for ROCm base" + if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then + docker buildx use "${BUILDER_NAME}" + else + docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use + fi + docker buildx inspect --bootstrap +} + +compute_base_content_hash() { + local use_sccache="$1" + local base_image_digest="$2" + local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" + local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" + local -a content_paths=() + local -a content_arg_names=() + + read -r -a content_paths <<< "${content_files}" + read -r -a content_arg_names <<< "${content_args}" + + { + printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")" + printf 'dockerfile:%s\n' "${DOCKERFILE}" + printf 'resolved-build-args:\n' + hash_rocm_base_arg_values \ + "${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}" + } | sha256sum | cut -d' ' -f1 +} + +build_base_image() { + local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}" + local base_hash="" + local build_date="" + local build_suffix="" + local base_image_arg="" + local base_image_digest="" + local rocm_version="" + local triton_arg="" + local pytorch_arg="" + local pytorch_vision_arg="" + local pytorch_audio_arg="" + local fa_arg="" + local aiter_arg="" + local mori_arg="" + local python_version_arg="" + local pytorch_rocm_arch_arg="" + local pytorch_branch="" + local aiter_branch="" + local dependency_summary="" + local descriptor="" + local ci_descriptor="" + local descriptive_tag="" + local stable_tag="${BASE_REPO}:base" + local ci_descriptive_tag="" + local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" + local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}" + local content_files_hash="" + local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}" + local -a tags=() + local -a no_cache_args=() + local -a sccache_args=() + local -a content_paths=() + + if [[ ! -f "${DOCKERFILE}" ]]; then + echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2 + exit 1 + fi + + build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}" + if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then + build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}" + fi + base_image_arg="$(extract_arg_default BASE_IMAGE)" + base_image_digest="$(resolve_image_digest "${base_image_arg}")" + read -r -a content_paths <<< "${content_files}" + content_files_hash="$(compute_content_hash "${content_paths[@]}")" + base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}") + rocm_version="$(rocm_version_from_base_image "${base_image_arg}")" + triton_arg="$(extract_arg_default TRITON_BRANCH)" + pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)" + pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)" + pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)" + fa_arg="$(extract_arg_default FA_BRANCH)" + aiter_arg="$(extract_arg_default AITER_BRANCH)" + mori_arg="$(extract_arg_default MORI_BRANCH)" + python_version_arg="$(extract_arg_default PYTHON_VERSION)" + pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)" + pytorch_branch="$(tag_component "${pytorch_arg}" 16)" + aiter_branch="$(tag_component "${aiter_arg}" 24)" + dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}" + descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")" + ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")" + + descriptive_tag="${BASE_REPO}:${descriptor}" + ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}" + + tags=(-t "${descriptive_tag}") + if should_push_stable_tag; then + tags+=(-t "${stable_tag}") + metadata_set "rocm-base-push-stable-tag" "1" + else + metadata_set "rocm-base-push-stable-tag" "0" + fi + + if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then + no_cache_args=(--no-cache) + fi + + for env_name in \ + SCCACHE_DOWNLOAD_URL \ + SCCACHE_ENDPOINT \ + SCCACHE_BUCKET_NAME \ + SCCACHE_REGION_NAME \ + SCCACHE_S3_NO_CREDENTIALS; do + if [[ -n "${!env_name:-}" ]]; then + sccache_args+=(--build-arg "${env_name}=${!env_name}") + fi + done + + echo "--- :docker: Building ROCm base image" + echo "Dockerfile: ${DOCKERFILE}" + echo "Descriptive tag: ${descriptive_tag}" + echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))" + echo "Content hash: ${base_hash}" + echo "Dependency summary: ${dependency_summary}" + echo "USE_SCCACHE: ${use_sccache}" + + docker buildx build \ + "${no_cache_args[@]}" \ + --pull \ + --progress "${BUILDKIT_PROGRESS:-plain}" \ + --file "${DOCKERFILE}" \ + --build-arg "USE_SCCACHE=${use_sccache}" \ + "${sccache_args[@]}" \ + --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \ + --label "org.opencontainers.image.vendor=vLLM" \ + --label "org.opencontainers.image.title=vLLM ROCm base" \ + --label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \ + --label "vllm.rocm_base.metadata_version=${metadata_version}" \ + --label "vllm.rocm_base.content_hash=${base_hash}" \ + --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \ + --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \ + --label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \ + --label "vllm.rocm_base.image.stable=${stable_tag}" \ + --label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \ + --label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \ + --label "vllm.rocm_base.descriptor=${descriptor}" \ + --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \ + --label "vllm.rocm_base.base_image=${base_image_arg}" \ + --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \ + --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \ + --label "vllm.rocm_base.dependency.python=${python_version_arg}" \ + --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \ + --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \ + --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \ + --label "vllm.rocm_base.dependency.triton=${triton_arg}" \ + --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \ + --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \ + --label "vllm.rocm_base.dependency.mori=${mori_arg}" \ + --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \ + "${tags[@]}" \ + --push \ + . + + docker buildx imagetools inspect "${descriptive_tag}" >/dev/null + + metadata_set "rocm-base-refresh" "1" + metadata_set "rocm-base-image" "${descriptive_tag}" + metadata_set "rocm-base-image-descriptive" "${descriptive_tag}" + metadata_set "rocm-base-image-stable" "${stable_tag}" + metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}" + metadata_set "rocm-base-metadata-version" "${metadata_version}" + metadata_set "rocm-base-content-hash" "${base_hash}" + metadata_set "rocm-base-content-files-hash" "${content_files_hash}" + metadata_set "rocm-base-content-files" "${content_files}" + metadata_set "rocm-base-content-args" "${content_args}" + metadata_set "rocm-base-base-image-digest" "${base_image_digest}" + metadata_set "rocm-base-dockerfile" "${DOCKERFILE}" + metadata_set "rocm-base-descriptor" "${descriptor}" + metadata_set "rocm-base-dependency-summary" "${dependency_summary}" + metadata_set "rocm-base-dependency-rocm" "${rocm_version}" + metadata_set "rocm-base-dependency-python" "${python_version_arg}" + metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}" + metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}" + metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}" + metadata_set "rocm-base-dependency-triton" "${triton_arg}" + metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}" + metadata_set "rocm-base-dependency-aiter" "${aiter_arg}" + metadata_set "rocm-base-dependency-mori" "${mori_arg}" + metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}" + metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}" + + echo "--- :white_check_mark: ROCm base image published" + echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds" +} + +main() { + metadata_set "rocm-base-refresh" "0" + + if ! rocm_base_changed; then + echo "ROCm base Dockerfile did not change; skipping base image refresh" + return 0 + fi + + setup_builder + build_base_image +} + +main "$@" diff --git a/.buildkite/scripts/rocm/smoke-test-image.sh b/.buildkite/scripts/rocm/smoke-test-image.sh new file mode 100755 index 0000000000..ed511c9b77 --- /dev/null +++ b/.buildkite/scripts/rocm/smoke-test-image.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Fast structural smoke test for the full ROCm CI image. + +set -euo pipefail + +image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}" + +docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec ' + if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi + if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi + if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi + if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi + + command -v python3 + command -v uv + command -v pytest + + if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then + echo No ROCm CLI found in image >&2 + exit 1 + fi + + python3 - <