[CI][ROCm] Add opt-in TheRock builds for AMD CI (#56351)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-16 00:09:45 -05:00
committed by GitHub
co-authored by Codex
parent af1c01499b
commit 711768fc17
5 changed files with 188 additions and 20 deletions
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
# Shared Dockerfile selection for the normal AMD CI build chain.
# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
# Custom recipes must preserve the CI targets and artifact layout. Additional
# COPY inputs also need the corresponding *_CONTENT_FILES overrides.
rocm_base_dockerfile() {
local default="docker/Dockerfile.rocm_base"
printf '%s\n' "${CI_ROCM_DOCKERFILE_BASE:-${ROCM_BASE_DOCKERFILE:-${default}}}"
}
rocm_ci_dockerfile() {
local default="docker/Dockerfile.rocm"
printf '%s\n' "${CI_ROCM_DOCKERFILE:-${CI_BASE_DOCKERFILE:-${default}}}"
}
using_custom_rocm_dockerfiles() {
[[ "$(rocm_base_dockerfile)" != "docker/Dockerfile.rocm_base" \
|| "$(rocm_ci_dockerfile)" != "docker/Dockerfile.rocm" ]]
}
rocm_dockerfile_stages() {
awk '
toupper($1) == "FROM" {
for (idx = 2; idx < NF; idx++) {
if (tolower($idx) == "as") print tolower($(idx + 1))
}
}
' "$1"
}
validate_rocm_dockerfile() {
local dockerfile="$1"
local stage=""
local stages=""
shift
case "${dockerfile}" in
/*|../*|*/../*|*/..|*[[:space:]]*)
echo "ROCm Dockerfile must be a repository-relative path without '..' or whitespace: ${dockerfile}" >&2
return 2
;;
esac
if [[ ! -f "${dockerfile}" ]]; then
echo "ROCm Dockerfile not found: ${dockerfile}" >&2
return 2
fi
if (($#)) && ! awk '
toupper($1) == "FROM" {
named = 0
for (idx = 2; idx < NF; idx++) {
if (tolower($idx) == "as") named = 1
}
if (!named) exit 1
}
' "${dockerfile}"; then
echo "ROCm Dockerfile must name every build stage: ${dockerfile}" >&2
return 2
fi
stages=$(rocm_dockerfile_stages "${dockerfile}")
for stage in "$@"; do
if ! grep -Fxq -- "${stage}" <<< "${stages}"; then
echo "ROCm Dockerfile ${dockerfile} is missing required stage: ${stage}" >&2
return 2
fi
done
}
configure_rocm_build() {
ROCM_BASE_DOCKERFILE=$(rocm_base_dockerfile)
CI_BASE_DOCKERFILE=$(rocm_ci_dockerfile)
export ROCM_BASE_DOCKERFILE CI_BASE_DOCKERFILE
using_custom_rocm_dockerfiles || return 0
# Custom stacks may supply dependencies without separate build stages.
export ROCM_DEP_CACHE_EXPORT_MODE=never
# Preserve the standard runtime tags, which are unique to each build,
# while preventing experimental builds from promoting stable images.
export ROCM_BASE_PUSH_STABLE_TAG=0
export CI_BASE_PUSH_STABLE_TAG=0
echo "Custom ROCm Dockerfiles: ${ROCM_BASE_DOCKERFILE} and ${CI_BASE_DOCKERFILE}"
}
+25 -4
View File
@@ -4,6 +4,10 @@
set -euo pipefail
# shellcheck source=.buildkite/scripts/rocm/build-config.sh
source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh"
configure_rocm_build
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
@@ -12,7 +16,12 @@ DEFAULT_ROCM_BASE_METADATA_VERSION="2"
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
ROCM_BASE_LAYER_CACHE_REF=""
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
BASE_CACHE_PREFIX="rocm-base"
if using_custom_rocm_dockerfiles; then
BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \
"${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)"
fi
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main"
ROCM_BASE_STABLE_TAG_UPDATED=0
declare -a ROCM_BASE_CACHE_ARGS=()
@@ -120,7 +129,7 @@ configure_rocm_base_layer_cache() {
fi
scope=$(rocm_base_layer_cache_scope)
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}"
ROCM_BASE_CACHE_ARGS+=(
--cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
)
@@ -139,7 +148,7 @@ configure_rocm_base_layer_cache() {
extract_arg_default() {
local arg_name="$1"
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
"${DOCKERFILE}" | head -1
}
@@ -419,11 +428,16 @@ build_base_image() {
read -r -a content_paths <<< "${content_files}"
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
rocm_version="$(extract_arg_default ROCM_SDK_VERSION)"
rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}"
triton_arg="$(extract_arg_default TRITON_BRANCH)"
pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}"
pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}"
pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}"
pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}"
fa_arg="$(extract_arg_default FA_BRANCH)"
aiter_arg="$(extract_arg_default AITER_BRANCH)"
mori_arg="$(extract_arg_default MORI_BRANCH)"
@@ -569,6 +583,13 @@ build_base_image() {
}
main() {
if using_custom_rocm_dockerfiles; then
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2
return 2
fi
validate_rocm_dockerfile "${DOCKERFILE}" || return $?
fi
metadata_set "rocm-base-refresh" "0"
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then