diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 15f8274fc6..6c5f3308b7 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -540,6 +540,26 @@ prepare_ci_build_context() { echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}" } +configure_custom_rocm_stages() { + using_custom_rocm_dockerfiles || return 0 + local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}" + local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}" + local stages="" + + (cd "${context_root}" && \ + validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \ + ci_base test export_vllm export_test_smoke csrc-build rust-build) \ + || return $? + # Hash all stages for custom layouts, including additional helper stages. + stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ') + CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}" + ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}" + ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}" + if [[ " ${stages} " != *" build_rocshmem "* ]]; then + unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY + fi +} + compose_dependency_cache_key() { local prefix="$1" local material="$2" @@ -569,11 +589,11 @@ hash_dockerfile_stages() { } emit = 0 } - $1 == "FROM" { + toupper($1) == "FROM" { stage = "" for (idx = 1; idx <= NF; idx++) { if (tolower($idx) == "as" && idx < NF) { - stage = $(idx + 1) + stage = tolower($(idx + 1)) } } emit = (stage in wanted) @@ -613,11 +633,11 @@ discover_dockerfile_stage_args() { } { line = $0 - if ($1 == "FROM") { + if (toupper($1) == "FROM") { stage = "" for (idx = 1; idx <= NF; idx++) { if (tolower($idx) == "as" && idx < NF) { - stage = $(idx + 1) + stage = tolower($(idx + 1)) } } emit = (stage in wanted) @@ -630,7 +650,7 @@ discover_dockerfile_stage_args() { for (idx = 1; idx <= line_count; idx++) { line = lines[idx] arg_name = line - sub(/^[[:space:]]*ARG[[:space:]]+/, "", arg_name) + sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name) if (arg_name != line) { sub(/[=[:space:]].*/, "", arg_name) if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) { @@ -716,7 +736,7 @@ extract_dockerfile_arg_default() { if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}" fi - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ "${physical_dockerfile}" | head -1 } @@ -1000,7 +1020,21 @@ create_and_bootstrap_builder() { } init_config() { + # shellcheck source=.buildkite/scripts/rocm/build-config.sh + source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh" + configure_rocm_build + TARGET="${1:-test-ci}" + if using_custom_rocm_dockerfiles; then + if [[ -z "${BASE_IMAGE:-}" ]]; then + echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2 + return 1 + fi + if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then + echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2 + return 1 + fi + fi BAKE_TARGETS=("${TARGET}") DEPENDENCY_CACHE_TARGETS=() CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}" @@ -1010,10 +1044,12 @@ init_config() { PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}" CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}" CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}" - CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" + if ! using_custom_rocm_dockerfiles; then + CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" + fi CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}" - export PYTORCH_ROCM_ARCH + export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX) CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl" @@ -1042,6 +1078,11 @@ print_header() { } validate_inputs() { + if using_custom_rocm_dockerfiles; then + validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $? + validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \ + ci_base test export_vllm export_test_smoke csrc-build rust-build || return $? + fi if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}" echo "Make sure you're running from the vLLM repository root" @@ -1069,6 +1110,11 @@ load_ci_hcl() { init_bake_files() { BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}") + if using_custom_rocm_dockerfiles; then + # Custom stacks reuse local cache without replacing the + # standard ROCm registry caches for the same commit or branch. + BAKE_FILES+=(--set '*.cache-to=') + fi } compute_ci_base_hash_if_needed() { @@ -1804,7 +1850,7 @@ compute_rocm_csrc_content_hash() { local -a content_args=() bake_dir=$(dirname "${VLLM_BAKE_FILE}") - dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" read -r -a content_paths <<< "${content_files}" mapfile -t content_args < <( get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}" @@ -1853,7 +1899,7 @@ compute_rocm_rust_content_hash() { local -a content_args=() bake_dir=$(dirname "${VLLM_BAKE_FILE}") - dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" read -r -a content_paths <<< "${content_files}" mapfile -t content_args < <( get_content_arg_names \ @@ -1943,7 +1989,7 @@ write_rocm_build_arg_override() { local arg_value="" bake_dir=$(dirname "${VLLM_BAKE_FILE}") - dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" mapfile -t arg_names < <( { get_content_arg_names \ @@ -2259,7 +2305,7 @@ extract_dependency_pins() { local val="" bake_dir=$(dirname "${VLLM_BAKE_FILE}") - dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" physical_dockerfile="${dockerfile_rocm}" if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}" @@ -2275,12 +2321,12 @@ extract_dependency_pins() { fi val=$( - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ "${physical_dockerfile}" | head -1 ) if [[ -n "${val}" ]]; then export "${var}=${val}" - echo "Extracted ${var}=${val} from Dockerfile.rocm" + echo "Extracted ${var}=${val} from ${dockerfile_rocm}" fi done } @@ -2297,7 +2343,7 @@ compute_dependency_cache_keys() { local deepep_material="" bake_dir=$(dirname "${VLLM_BAKE_FILE}") - dockerfile_rocm="${bake_dir}/Dockerfile.rocm" + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH") ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH") rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH") @@ -2687,6 +2733,7 @@ main() { init_bake_files if is_ci_base_target; then prepare_ci_build_context + configure_custom_rocm_stages fi compute_ci_base_hash_if_needed configure_ci_base_image_refs @@ -2697,6 +2744,7 @@ main() { # version metadata only after that lookup sees the available tag history. if ! is_ci_base_target; then prepare_ci_build_context + configure_custom_rocm_stages fi extract_dependency_pins write_rocm_build_arg_override diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 62fda27bf0..541a8f5525 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -34,6 +34,9 @@ ############################################################################### set -o pipefail +# shellcheck source=.buildkite/scripts/rocm/build-config.sh +source "$(dirname "${BASH_SOURCE[0]}")/../rocm/build-config.sh" || exit $? + : "${BUILDKIT_PROGRESS:=plain}" : "${TERM:=xterm-256color}" : "${FORCE_COLOR:=1}" @@ -106,6 +109,10 @@ clear_ci_orchestration_env() { VLLM_CI_REQUIRE_WORKSPACE_MOUNT \ VLLM_TEST_COMMANDS \ VLLM_CI_BRANCH \ + CI_ROCM_DOCKERFILE_BASE \ + CI_ROCM_DOCKERFILE \ + ROCM_BASE_DOCKERFILE \ + CI_BASE_DOCKERFILE \ VLLM_CI_BASE_IMAGE \ VLLM_CI_FALLBACK_IMAGE \ VLLM_CI_DOCKER_DISABLED \ @@ -225,6 +232,9 @@ prepare_artifact_image() { metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1) if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then base_image=$(tr -d '[:space:]' < "${metadata_file}") + elif using_custom_rocm_dockerfiles; then + echo "Custom ROCm ci_base metadata is missing; using the full CI image" + return 1 fi echo "--- Preparing local ROCm test image" diff --git a/.buildkite/scripts/rocm/build-config.sh b/.buildkite/scripts/rocm/build-config.sh new file mode 100644 index 0000000000..ad0179b163 --- /dev/null +++ b/.buildkite/scripts/rocm/build-config.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Shared Dockerfile selection for the normal AMD CI build chain. +# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths. +# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and +# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock. +# Custom recipes must preserve the CI targets and artifact layout. Additional +# COPY inputs also need the corresponding *_CONTENT_FILES overrides. + +rocm_base_dockerfile() { + local default="docker/Dockerfile.rocm_base" + printf '%s\n' "${CI_ROCM_DOCKERFILE_BASE:-${ROCM_BASE_DOCKERFILE:-${default}}}" +} + +rocm_ci_dockerfile() { + local default="docker/Dockerfile.rocm" + printf '%s\n' "${CI_ROCM_DOCKERFILE:-${CI_BASE_DOCKERFILE:-${default}}}" +} + +using_custom_rocm_dockerfiles() { + [[ "$(rocm_base_dockerfile)" != "docker/Dockerfile.rocm_base" \ + || "$(rocm_ci_dockerfile)" != "docker/Dockerfile.rocm" ]] +} + +rocm_dockerfile_stages() { + awk ' + toupper($1) == "FROM" { + for (idx = 2; idx < NF; idx++) { + if (tolower($idx) == "as") print tolower($(idx + 1)) + } + } + ' "$1" +} + +validate_rocm_dockerfile() { + local dockerfile="$1" + local stage="" + local stages="" + shift + + case "${dockerfile}" in + /*|../*|*/../*|*/..|*[[:space:]]*) + echo "ROCm Dockerfile must be a repository-relative path without '..' or whitespace: ${dockerfile}" >&2 + return 2 + ;; + esac + if [[ ! -f "${dockerfile}" ]]; then + echo "ROCm Dockerfile not found: ${dockerfile}" >&2 + return 2 + fi + if (($#)) && ! awk ' + toupper($1) == "FROM" { + named = 0 + for (idx = 2; idx < NF; idx++) { + if (tolower($idx) == "as") named = 1 + } + if (!named) exit 1 + } + ' "${dockerfile}"; then + echo "ROCm Dockerfile must name every build stage: ${dockerfile}" >&2 + return 2 + fi + stages=$(rocm_dockerfile_stages "${dockerfile}") + for stage in "$@"; do + if ! grep -Fxq -- "${stage}" <<< "${stages}"; then + echo "ROCm Dockerfile ${dockerfile} is missing required stage: ${stage}" >&2 + return 2 + fi + done +} + +configure_rocm_build() { + ROCM_BASE_DOCKERFILE=$(rocm_base_dockerfile) + CI_BASE_DOCKERFILE=$(rocm_ci_dockerfile) + export ROCM_BASE_DOCKERFILE CI_BASE_DOCKERFILE + using_custom_rocm_dockerfiles || return 0 + + # Custom stacks may supply dependencies without separate build stages. + export ROCM_DEP_CACHE_EXPORT_MODE=never + + # Preserve the standard runtime tags, which are unique to each build, + # while preventing experimental builds from promoting stable images. + export ROCM_BASE_PUSH_STABLE_TAG=0 + export CI_BASE_PUSH_STABLE_TAG=0 + echo "Custom ROCm Dockerfiles: ${ROCM_BASE_DOCKERFILE} and ${CI_BASE_DOCKERFILE}" +} diff --git a/.buildkite/scripts/rocm/refresh-base-image.sh b/.buildkite/scripts/rocm/refresh-base-image.sh index 0e10b98c9e..10e594577a 100755 --- a/.buildkite/scripts/rocm/refresh-base-image.sh +++ b/.buildkite/scripts/rocm/refresh-base-image.sh @@ -4,6 +4,10 @@ set -euo pipefail +# shellcheck source=.buildkite/scripts/rocm/build-config.sh +source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh" +configure_rocm_build + DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}" BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}" CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}" @@ -12,7 +16,12 @@ DEFAULT_ROCM_BASE_METADATA_VERSION="2" DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}" ROCM_BASE_LAYER_CACHE_REF="" -ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main" +BASE_CACHE_PREFIX="rocm-base" +if using_custom_rocm_dockerfiles; then + BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \ + "${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)" +fi +ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main" ROCM_BASE_STABLE_TAG_UPDATED=0 declare -a ROCM_BASE_CACHE_ARGS=() @@ -120,7 +129,7 @@ configure_rocm_base_layer_cache() { fi scope=$(rocm_base_layer_cache_scope) - ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}" + ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}" ROCM_BASE_CACHE_ARGS+=( --cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}" ) @@ -139,7 +148,7 @@ configure_rocm_base_layer_cache() { extract_arg_default() { local arg_name="$1" - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ "${DOCKERFILE}" | head -1 } @@ -419,11 +428,16 @@ build_base_image() { read -r -a content_paths <<< "${content_files}" content_files_hash="$(compute_content_hash "${content_paths[@]}")" base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}") - rocm_version="$(rocm_version_from_base_image "${base_image_arg}")" + rocm_version="$(extract_arg_default ROCM_SDK_VERSION)" + rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}" triton_arg="$(extract_arg_default TRITON_BRANCH)" pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)" pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)" pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)" + triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}" + pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}" + pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}" + pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}" fa_arg="$(extract_arg_default FA_BRANCH)" aiter_arg="$(extract_arg_default AITER_BRANCH)" mori_arg="$(extract_arg_default MORI_BRANCH)" @@ -569,6 +583,13 @@ build_base_image() { } main() { + if using_custom_rocm_dockerfiles; then + if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then + echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2 + return 2 + fi + validate_rocm_dockerfile "${DOCKERFILE}" || return $? + fi metadata_set "rocm-base-refresh" "0" if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then diff --git a/docker/docker-bake-rocm.hcl b/docker/docker-bake-rocm.hcl index 9d73a6505e..ab523a296a 100644 --- a/docker/docker-bake-rocm.hcl +++ b/docker/docker-bake-rocm.hcl @@ -33,6 +33,10 @@ variable "CI_BASE_CONTENT_HASH" { default = "" } +variable "CI_BASE_DOCKERFILE" { + default = "docker/Dockerfile.rocm" +} + # REMOTE_VLLM=0: use local source via Docker build context (ONBUILD COPY ./ vllm/) # REMOTE_VLLM=1: clone from GitHub at VLLM_BRANCH (standalone builds without local source) variable "REMOTE_VLLM" { @@ -74,7 +78,7 @@ group "default" { } target "_common-rocm" { - dockerfile = "docker/Dockerfile.rocm" + dockerfile = CI_BASE_DOCKERFILE context = "." args = { max_jobs = MAX_JOBS