[ROCm][CI] Reuse equivalent ROCm CI images (#48646)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
f18e10a7e1
commit
7f6432cc0d
@@ -2,11 +2,13 @@ name: vllm_rocm_ci
|
||||
job_dirs:
|
||||
- ".buildkite/hardware_tests"
|
||||
run_all_patterns:
|
||||
- ".dockerignore"
|
||||
- "docker/Dockerfile.rocm"
|
||||
- "docker/Dockerfile.rocm_base"
|
||||
- "docker/ci-rocm.hcl"
|
||||
- "docker/docker-bake-rocm.hcl"
|
||||
- ".buildkite/hardware_tests/amd.yaml"
|
||||
- ".buildkite/ci_config_rocm.yaml"
|
||||
- ".buildkite/scripts/ci-bake-rocm.sh"
|
||||
- ".buildkite/scripts/rocm/"
|
||||
- ".buildkite/scripts/hardware_ci/run-amd-test.py"
|
||||
@@ -14,9 +16,20 @@ run_all_patterns:
|
||||
- "CMakeLists.txt"
|
||||
- "requirements/common.txt"
|
||||
- "requirements/rocm.txt"
|
||||
- "requirements/build/rust.txt"
|
||||
- "requirements/build/rocm.txt"
|
||||
- "requirements/test/rocm.txt"
|
||||
- "pyproject.toml"
|
||||
- "setup.py"
|
||||
- "build_rust.sh"
|
||||
- "rust/"
|
||||
- "rust-toolchain.toml"
|
||||
- "tools/build_rust.py"
|
||||
- "tools/install_protoc.sh"
|
||||
- "tools/install_torchcodec_rocm.sh"
|
||||
- "tests/vllm_test_utils/"
|
||||
- "vllm/envs.py"
|
||||
- "vllm/__init__.py"
|
||||
- "csrc/"
|
||||
- "cmake/"
|
||||
run_all_exclude_patterns:
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
group: Hardware - AMD Build
|
||||
|
||||
# ROCm image flow:
|
||||
# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
|
||||
# 2. Build ci_base from either the stable base or the freshly refreshed base.
|
||||
# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
|
||||
# 1. Select the content-matching ROCm base, building only on a cache miss.
|
||||
# 2. Build ci_base from either the stable base or the newly selected base.
|
||||
# 3. Build the build-scoped ROCm CI image and smoke-test it before GPU jobs run.
|
||||
steps:
|
||||
- label: "AMD: :docker: refresh ROCm base"
|
||||
key: refresh-rocm-base-amd
|
||||
@@ -12,6 +12,8 @@ steps:
|
||||
no_plugin: true
|
||||
commands:
|
||||
- bash .buildkite/scripts/rocm/refresh-base-image.sh
|
||||
concurrency: 1
|
||||
concurrency_group: "vllm/rocm/base-build/$BUILDKITE_COMMIT"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
BUILDKIT_PROGRESS: "tty"
|
||||
@@ -38,18 +40,21 @@ steps:
|
||||
no_plugin: true
|
||||
commands:
|
||||
- bash .buildkite/scripts/rocm/build-ci-base.sh
|
||||
concurrency: 1
|
||||
concurrency_group: "vllm/rocm/ci-base-build/$BUILDKITE_COMMIT"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
BUILDKIT_PROGRESS: "tty"
|
||||
TERM: "xterm-256color"
|
||||
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
|
||||
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
|
||||
REMOTE_VLLM: "1"
|
||||
VLLM_BRANCH: "$BUILDKITE_COMMIT"
|
||||
REMOTE_VLLM: "0"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 1 # Transient Docker/BuildKit failure
|
||||
limit: 1
|
||||
- exit_status: 2 # Exact image found but registry handoff failed
|
||||
limit: 1
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: -10 # Agent was lost
|
||||
@@ -71,9 +76,8 @@ steps:
|
||||
TERM: "xterm-256color"
|
||||
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
|
||||
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
|
||||
IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
|
||||
REMOTE_VLLM: "1"
|
||||
VLLM_BRANCH: "$BUILDKITE_COMMIT"
|
||||
IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"
|
||||
REMOTE_VLLM: "0"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 1 # Transient Docker/BuildKit failure
|
||||
|
||||
+707
-487
File diff suppressed because it is too large
Load Diff
@@ -36,6 +36,19 @@ set -o pipefail
|
||||
: "${ROCM_DOCKER_TTY:=1}"
|
||||
: "${PYTHONFAULTHANDLER:=1}"
|
||||
: "${PYTEST_TIMEOUT:=2400}"
|
||||
|
||||
# Preserve runner-only values before clear_ci_orchestration_env removes them
|
||||
# from the environment inherited by the test process.
|
||||
amd_diagnostics_expected_gpu_count="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}"
|
||||
amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}"
|
||||
amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}"
|
||||
amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}"
|
||||
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}"
|
||||
amd_diagnostics_collected=0
|
||||
amd_diagnostics_probe_budget_seconds=25
|
||||
amd_diagnostics_command_timeout_seconds=5
|
||||
amd_diagnostics_upload_timeout_seconds=20
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
|
||||
fi
|
||||
@@ -84,9 +97,11 @@ clear_ci_orchestration_env() {
|
||||
VLLM_CI_BASE_IMAGE \
|
||||
VLLM_CI_FALLBACK_IMAGE \
|
||||
VLLM_CI_DOCKER_DISABLED \
|
||||
VLLM_CI_DIAGNOSTICS_DIR \
|
||||
VLLM_CI_ARTIFACT_GLOB \
|
||||
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
|
||||
VLLM_CI_EXPECTED_GPU_COUNT \
|
||||
VLLM_CI_K8S_NODE_NAME \
|
||||
VLLM_CI_USE_ARTIFACTS \
|
||||
VLLM_CI_RESULTS_ROOT \
|
||||
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
|
||||
@@ -350,6 +365,11 @@ prepare_native_workspace() {
|
||||
echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${BUILDKITE_BUILD_ID:-}" \
|
||||
|| "${recorded_base}" != *":ci_base-build-${BUILDKITE_BUILD_ID}" ]]; then
|
||||
echo "ROCm artifact base is not scoped to Buildkite build ${BUILDKITE_BUILD_ID:-unset}: ${recorded_base}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
|
||||
echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
|
||||
return 1
|
||||
@@ -495,12 +515,220 @@ is_multi_node() {
|
||||
return 1
|
||||
}
|
||||
|
||||
run_amd_diagnostic() {
|
||||
local log_file=$1
|
||||
local probe_deadline=$2
|
||||
shift 2
|
||||
local command_status=0
|
||||
local command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}
|
||||
local remaining_seconds=$((probe_deadline - SECONDS))
|
||||
|
||||
{
|
||||
printf '\n$'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
} | tee -a "${log_file}"
|
||||
|
||||
if [[ "${remaining_seconds}" -le 0 ]]; then
|
||||
echo "[diagnostic probe budget exhausted; command skipped]" \
|
||||
| tee -a "${log_file}"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then
|
||||
command_timeout_seconds=${remaining_seconds}
|
||||
fi
|
||||
|
||||
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \
|
||||
| tee -a "${log_file}"
|
||||
command_status=${PIPESTATUS[0]}
|
||||
if [[ "${command_status}" -ne 0 ]]; then
|
||||
echo "[diagnostic command exited with status ${command_status}]" \
|
||||
| tee -a "${log_file}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
collect_rocm_failure_diagnostics() {
|
||||
local exit_code=$1
|
||||
local job_id="${BUILDKITE_JOB_ID:-local}"
|
||||
local retry_count="${BUILDKITE_RETRY_COUNT:-0}"
|
||||
local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}"
|
||||
local diagnostics_relative_path=""
|
||||
local diagnostics_path=""
|
||||
local upload_root="${amd_diagnostics_checkout_root}"
|
||||
local upload_path=""
|
||||
local exit_signal=""
|
||||
local probe_deadline=0
|
||||
|
||||
if [[ "${amd_diagnostics_collected}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
amd_diagnostics_collected=1
|
||||
|
||||
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
|
||||
echo "Skipping AMD GPU diagnostics for a CPU-only job."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! command -v timeout >/dev/null 2>&1; then
|
||||
echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job."
|
||||
return 0
|
||||
fi
|
||||
|
||||
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
|
||||
retry_count="${retry_count//[^A-Za-z0-9_.-]/_}"
|
||||
parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}"
|
||||
|
||||
# Buildkite keeps the supplied artifact path. Restrict the configurable
|
||||
# directory to a checkout-relative path so the UI shows a clean artifact key.
|
||||
if [[ -z "${amd_diagnostics_dir}" \
|
||||
|| "${amd_diagnostics_dir}" == /* \
|
||||
|| "${amd_diagnostics_dir}" == *".."* ]]; then
|
||||
echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR"
|
||||
amd_diagnostics_dir="artifacts/amd-gpu-diagnostics"
|
||||
fi
|
||||
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log"
|
||||
diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}"
|
||||
upload_path="${diagnostics_relative_path}"
|
||||
|
||||
if ! mkdir -p "$(dirname "${diagnostics_path}")" \
|
||||
|| ! : > "${diagnostics_path}"; then
|
||||
diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || {
|
||||
echo "WARNING: unable to create an AMD GPU diagnostics log."
|
||||
return 0
|
||||
}
|
||||
upload_root=$(dirname "${diagnostics_path}")
|
||||
upload_path=$(basename "${diagnostics_path}")
|
||||
fi
|
||||
|
||||
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
|
||||
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
{
|
||||
echo "AMD GPU failure diagnostics"
|
||||
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "exit_code=${exit_code}"
|
||||
echo "exit_signal=${exit_signal:-none}"
|
||||
echo "build_url=${BUILDKITE_BUILD_URL:-unknown}"
|
||||
echo "job_id=${BUILDKITE_JOB_ID:-unknown}"
|
||||
echo "step_key=${BUILDKITE_STEP_KEY:-unknown}"
|
||||
echo "step_label=${BUILDKITE_LABEL:-unknown}"
|
||||
echo "test_group=${amd_diagnostics_test_group}"
|
||||
echo "retry_count=${retry_count}"
|
||||
echo "parallel_job=${parallel_job}"
|
||||
echo "execution_mode=${amd_diagnostics_execution_mode}"
|
||||
echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"
|
||||
echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"
|
||||
echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"
|
||||
echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}"
|
||||
echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"
|
||||
echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}"
|
||||
echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)"
|
||||
} > "${diagnostics_path}"
|
||||
|
||||
echo "--- :rotating_light: AMD GPU failure diagnostics"
|
||||
cat "${diagnostics_path}"
|
||||
|
||||
probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds))
|
||||
|
||||
# Collect the most useful evidence first so auxiliary probes cannot consume
|
||||
# the shared deadline before GPU health is captured.
|
||||
echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}"
|
||||
if command -v amd-smi >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version
|
||||
# Bus data identifies a card within the public node without publishing its
|
||||
# persistent UUID, serial number, or process list.
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all
|
||||
elif command -v rocm-smi >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
|
||||
--showbus --showreplaycount --showrasinfo --showpagesinfo
|
||||
else
|
||||
echo "Neither amd-smi nor rocm-smi is available." \
|
||||
| tee -a "${diagnostics_path}"
|
||||
fi
|
||||
|
||||
echo "Node resource diagnostics (runner-visible scope)" \
|
||||
| tee -a "${diagnostics_path}"
|
||||
if command -v df >/dev/null 2>&1; then
|
||||
echo "checkout_filesystem:" | tee -a "${diagnostics_path}"
|
||||
if [[ -d "${amd_diagnostics_checkout_root}" ]]; then
|
||||
(
|
||||
cd "${amd_diagnostics_checkout_root}" || exit 0
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
|
||||
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
|
||||
-- .
|
||||
)
|
||||
else
|
||||
echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}"
|
||||
fi
|
||||
echo "tmp_filesystem:" | tee -a "${diagnostics_path}"
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
|
||||
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
|
||||
-- /tmp
|
||||
fi
|
||||
if command -v free >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h
|
||||
fi
|
||||
if [[ -r /sys/fs/cgroup/memory.events ]]; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \
|
||||
/sys/fs/cgroup/memory.events
|
||||
fi
|
||||
|
||||
echo "Unauthenticated outer-runner network reachability diagnostics" \
|
||||
| tee -a "${diagnostics_path}"
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
--silent --location --proto '=https' --proto-redir '=https' \
|
||||
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
|
||||
--write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
|
||||
https://huggingface.co/api/models/gpt2
|
||||
run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
|
||||
--silent --location --proto '=https' --proto-redir '=https' \
|
||||
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
|
||||
--write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
|
||||
'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack'
|
||||
else
|
||||
echo "curl unavailable; reachability probes skipped." \
|
||||
| tee -a "${diagnostics_path}"
|
||||
fi
|
||||
|
||||
if command -v buildkite-agent >/dev/null 2>&1 \
|
||||
&& [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then
|
||||
if (
|
||||
cd "${upload_root}" \
|
||||
&& BUILDKITE_AGENT_DEBUG=false \
|
||||
BUILDKITE_AGENT_DEBUG_HTTP=false \
|
||||
BUILDKITE_AGENT_TRACE_HTTP=false \
|
||||
BUILDKITE_AGENT_LOG_LEVEL=error \
|
||||
timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \
|
||||
buildkite-agent artifact upload "${upload_path}" \
|
||||
>/dev/null 2>&1
|
||||
); then
|
||||
echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}"
|
||||
else
|
||||
echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}"
|
||||
fi
|
||||
else
|
||||
echo "Buildkite agent unavailable; diagnostics artifact was not uploaded."
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
handle_pytest_exit() {
|
||||
local exit_code=$1
|
||||
if [ "$exit_code" -eq 5 ]; then
|
||||
echo "Pytest exit code 5 (no tests collected) - treating as success."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${exit_code}" -ne 0 ]]; then
|
||||
collect_rocm_failure_diagnostics "${exit_code}"
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
@@ -659,6 +887,19 @@ re_quote_pytest_markers() {
|
||||
echo "${output% }"
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
|
||||
handle_amd_runner_exit() {
|
||||
local exit_code=${1:-$?}
|
||||
if [[ "${exit_code}" -ne 0 ]]; then
|
||||
collect_rocm_failure_diagnostics "${exit_code}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup
|
||||
# traps below replace this trap but call the same handler before cleaning up.
|
||||
trap handle_amd_runner_exit EXIT
|
||||
|
||||
###############################################################################
|
||||
# Main
|
||||
###############################################################################
|
||||
@@ -667,7 +908,10 @@ if is_native_runtime; then
|
||||
echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
|
||||
artifact_work_dir=""
|
||||
|
||||
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
|
||||
cleanup_native_workspace() {
|
||||
local exit_code=$?
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
if [[ -n "${artifact_work_dir}" ]]; then
|
||||
rm -rf "${artifact_work_dir}"
|
||||
fi
|
||||
@@ -727,7 +971,10 @@ image_name="${VLLM_CI_FALLBACK_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:-local}}"
|
||||
artifact_work_dir=""
|
||||
container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
|
||||
|
||||
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
|
||||
remove_docker_container() {
|
||||
local exit_code=$?
|
||||
handle_amd_runner_exit "${exit_code}"
|
||||
if docker container inspect "${container_name}" >/dev/null 2>&1; then
|
||||
docker rm -f "${container_name}" || true
|
||||
fi
|
||||
|
||||
@@ -12,17 +12,33 @@ metadata_get() {
|
||||
|
||||
main() {
|
||||
local base_refreshed=""
|
||||
local selected_base=""
|
||||
|
||||
# The content identity below describes this checkout. Do not let legacy
|
||||
# external pipeline settings switch the build to a remote source tree.
|
||||
export REMOTE_VLLM=0
|
||||
unset VLLM_BRANCH
|
||||
|
||||
base_refreshed="$(metadata_get rocm-base-refresh)"
|
||||
selected_base="$(metadata_get rocm-base-image)"
|
||||
if [[ -n "${selected_base}" ]]; then
|
||||
if [[ ! "${selected_base}" =~ @sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "Selected ROCm base handoff is not digest-pinned: ${selected_base}" >&2
|
||||
return 1
|
||||
fi
|
||||
export BASE_IMAGE="${selected_base}"
|
||||
echo "Using selected ROCm base image for ci_base: ${BASE_IMAGE}"
|
||||
elif [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "Required ROCm base handoff metadata is missing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${base_refreshed}" == "1" ]]; then
|
||||
export BASE_IMAGE
|
||||
export CI_BASE_PUSH_STABLE_TAG
|
||||
|
||||
BASE_IMAGE="$(metadata_get rocm-base-image)"
|
||||
CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)"
|
||||
CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}"
|
||||
|
||||
echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}"
|
||||
echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the ROCm CI test image or wheel artifact.
|
||||
#
|
||||
# When Dockerfile.rocm_base changes, always build the full image so downstream
|
||||
# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain.
|
||||
# When base selection changes, build the full image so downstream ROCm tests
|
||||
# validate the selected base -> ci_base -> CI image chain.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -13,46 +13,55 @@ metadata_get() {
|
||||
fi
|
||||
}
|
||||
|
||||
use_ci_base_if_present() {
|
||||
local ci_base_image=""
|
||||
load_digest_handoff() {
|
||||
local metadata_key="$1"
|
||||
local env_name="$2"
|
||||
local description="$3"
|
||||
local image_ref=""
|
||||
|
||||
ci_base_image="$(metadata_get rocm-ci-base-image)"
|
||||
if [[ -z "${ci_base_image}" ]]; then
|
||||
image_ref="$(metadata_get "${metadata_key}")"
|
||||
if [[ -z "${image_ref}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ ! "${image_ref}" =~ @sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "${description} is not digest-pinned: ${image_ref}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
export CI_BASE_IMAGE="${ci_base_image}"
|
||||
echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
|
||||
}
|
||||
|
||||
use_refreshed_base_if_present() {
|
||||
local base_refreshed=""
|
||||
|
||||
base_refreshed="$(metadata_get rocm-base-refresh)"
|
||||
if [[ "${base_refreshed}" != "1" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
export BASE_IMAGE
|
||||
export IMAGE_TAG_LATEST
|
||||
|
||||
BASE_IMAGE="$(metadata_get rocm-base-image)"
|
||||
IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
|
||||
|
||||
echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
|
||||
if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
|
||||
echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
printf -v "${env_name}" '%s' "${image_ref}"
|
||||
export "${env_name?}"
|
||||
echo "Using ${description}: ${image_ref}"
|
||||
}
|
||||
|
||||
main() {
|
||||
local base_refreshed=0
|
||||
|
||||
use_ci_base_if_present || true
|
||||
# This job always builds the checked-out commit. Some externally generated
|
||||
# pipeline templates still inject remote-fetch settings; do not let those
|
||||
# settings make the source identity commit-specific or bypass local edits.
|
||||
export REMOTE_VLLM=0
|
||||
unset VLLM_BRANCH
|
||||
|
||||
if use_refreshed_base_if_present; then
|
||||
if ! load_digest_handoff \
|
||||
rocm-ci-base-image CI_BASE_IMAGE "ROCm ci_base handoff"; then
|
||||
if [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "Required ROCm ci_base handoff metadata is missing or invalid" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "No ROCm ci_base handoff metadata found; using the local default"
|
||||
fi
|
||||
|
||||
if ! load_digest_handoff \
|
||||
rocm-base-image BASE_IMAGE "ROCm base handoff"; then
|
||||
if [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "Required ROCm base handoff metadata is missing or invalid" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "No ROCm base handoff metadata found; using the local default"
|
||||
fi
|
||||
|
||||
if [[ "$(metadata_get rocm-base-refresh)" == "1" ]]; then
|
||||
echo "The selected ROCm base differs from the current stable base"
|
||||
base_refreshed=1
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes.
|
||||
#
|
||||
# Normal AMD CI builds should not pay for this path. The script no-ops unless
|
||||
# docker/Dockerfile.rocm_base changed relative to the branch base, the previous
|
||||
# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set.
|
||||
# Select the content-addressed ROCm base for this checkout, building it only
|
||||
# when the exact content ref is absent.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
|
||||
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
|
||||
CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}"
|
||||
CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
|
||||
BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}"
|
||||
DEFAULT_ROCM_BASE_METADATA_VERSION="1"
|
||||
DEFAULT_ROCM_BASE_METADATA_VERSION="2"
|
||||
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
|
||||
DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE"
|
||||
|
||||
ROCM_BASE_LAYER_CACHE_REF=""
|
||||
ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
|
||||
ROCM_BASE_STABLE_TAG_UPDATED=0
|
||||
declare -a ROCM_BASE_CACHE_ARGS=()
|
||||
|
||||
metadata_set() {
|
||||
local key="$1"
|
||||
@@ -21,7 +22,10 @@ metadata_set() {
|
||||
|
||||
[[ -n "${value}" ]] || return 0
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent meta-data set "${key}" "${value}" || true
|
||||
buildkite-agent meta-data set "${key}" "${value}"
|
||||
elif [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "buildkite-agent is required to publish ROCm base metadata" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -56,6 +60,82 @@ tag_component() {
|
||||
clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}"
|
||||
}
|
||||
|
||||
normalize_repo_slug() {
|
||||
local repo_slug="${1:-}"
|
||||
|
||||
repo_slug="${repo_slug%/}"
|
||||
repo_slug="${repo_slug%.git}"
|
||||
repo_slug="${repo_slug#https://github.com/}"
|
||||
repo_slug="${repo_slug#http://github.com/}"
|
||||
repo_slug="${repo_slug#ssh://git@github.com/}"
|
||||
repo_slug="${repo_slug#git@github.com:}"
|
||||
repo_slug="${repo_slug#github.com/}"
|
||||
printf '%s\n' "${repo_slug}"
|
||||
}
|
||||
|
||||
is_trusted_main_build() {
|
||||
local actual_repo=""
|
||||
local trusted_repo=""
|
||||
|
||||
[[ "${BUILDKITE:-false}" == "true" ]] || return 1
|
||||
[[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1
|
||||
[[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] || return 1
|
||||
actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}")
|
||||
trusted_repo=$(normalize_repo_slug \
|
||||
"${ROCM_BASE_STABLE_REPO_SLUG:-vllm-project/vllm}")
|
||||
[[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]]
|
||||
}
|
||||
|
||||
rocm_base_layer_cache_scope() {
|
||||
local pull_request="${BUILDKITE_PULL_REQUEST:-false}"
|
||||
local branch="${BUILDKITE_PULL_REQUEST_HEAD_BRANCH:-${BUILDKITE_BRANCH:-local}}"
|
||||
local identity=""
|
||||
local repo_slug=""
|
||||
|
||||
if is_trusted_main_build; then
|
||||
printf 'main\n'
|
||||
return 0
|
||||
fi
|
||||
if [[ "${pull_request}" != "false" && -n "${pull_request}" ]]; then
|
||||
repo_slug=$(normalize_repo_slug "${BUILDKITE_REPO:-local}")
|
||||
identity=$(printf '%s\n' "${repo_slug:-local}" | sha256sum | cut -c1-12)
|
||||
printf 'pr-%s-%s\n' \
|
||||
"$(tag_component "${pull_request}" 32)" "${identity}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
identity=$(printf '%s\n%s\n' "${BUILDKITE_REPO:-local}" "${branch}" \
|
||||
| sha256sum | cut -c1-12)
|
||||
printf 'preview-%s-%s\n' "$(tag_component "${branch}" 24)" "${identity}"
|
||||
}
|
||||
|
||||
configure_rocm_base_layer_cache() {
|
||||
local scope=""
|
||||
|
||||
ROCM_BASE_CACHE_ARGS=()
|
||||
if [[ "${ROCM_BASE_NO_CACHE:-0}" == "1" ]]; then
|
||||
ROCM_BASE_CACHE_ARGS+=(--no-cache)
|
||||
ROCM_BASE_LAYER_CACHE_REF="disabled"
|
||||
return 0
|
||||
fi
|
||||
|
||||
scope=$(rocm_base_layer_cache_scope)
|
||||
ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
|
||||
ROCM_BASE_CACHE_ARGS+=(
|
||||
--cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
|
||||
)
|
||||
if [[ "${ROCM_BASE_LAYER_CACHE_REF}" != \
|
||||
"${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" ]]; then
|
||||
ROCM_BASE_CACHE_ARGS+=(
|
||||
--cache-from "type=registry,ref=${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}"
|
||||
)
|
||||
fi
|
||||
ROCM_BASE_CACHE_ARGS+=(
|
||||
--cache-to \
|
||||
"type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF},mode=max,ignore-error=true"
|
||||
)
|
||||
}
|
||||
|
||||
extract_arg_default() {
|
||||
local arg_name="$1"
|
||||
|
||||
@@ -65,41 +145,123 @@ extract_arg_default() {
|
||||
|
||||
resolve_image_digest() {
|
||||
local image_ref="$1"
|
||||
local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
|
||||
local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
|
||||
local attempt=0
|
||||
local output=""
|
||||
local digest=""
|
||||
local status=0
|
||||
|
||||
docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
|
||||
| sed -n -E 's/^Digest:[[:space:]]+//p' \
|
||||
| head -1 || true
|
||||
}
|
||||
if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then
|
||||
printf '%s\n' "${BASH_REMATCH[1]}"
|
||||
return 0
|
||||
fi
|
||||
if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
|
||||
|| ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
|
||||
echo "Invalid image digest retry configuration" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
resolve_rocm_base_arg_value() {
|
||||
local arg_name="$1"
|
||||
local use_sccache="$2"
|
||||
|
||||
case "${arg_name}" in
|
||||
USE_SCCACHE)
|
||||
printf '%s\n' "${use_sccache}"
|
||||
;;
|
||||
*)
|
||||
extract_arg_default "${arg_name}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
hash_rocm_base_arg_values() {
|
||||
local use_sccache="$1"
|
||||
local base_image_digest="$2"
|
||||
local arg_name=""
|
||||
local arg_value=""
|
||||
shift 2 || true
|
||||
|
||||
for arg_name in "$@"; do
|
||||
[[ -n "${arg_name}" ]] || continue
|
||||
arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}")
|
||||
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
|
||||
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
|
||||
printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}"
|
||||
for ((attempt = 1; attempt <= attempts; attempt++)); do
|
||||
status=0
|
||||
output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$?
|
||||
digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}")
|
||||
if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
||||
printf '%s\n' "${digest}"
|
||||
return 0
|
||||
fi
|
||||
if ((attempt < attempts)); then
|
||||
printf \
|
||||
'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \
|
||||
"${image_ref}" "${attempt}" "${attempts}" "${status}" >&2
|
||||
sleep "${delay_secs}"
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'Failed to resolve digest for %s (status %d)\n%s\n' \
|
||||
"${image_ref}" "${status}" "${output:-<no output>}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
trusted_base_content_ref() {
|
||||
local base_hash="$1"
|
||||
local metadata_version="$2"
|
||||
|
||||
printf '%s:base-v%s-%s\n' "${BASE_REPO}" "${metadata_version}" "${base_hash}"
|
||||
}
|
||||
|
||||
scoped_base_content_ref() {
|
||||
local base_hash="$1"
|
||||
local metadata_version="$2"
|
||||
|
||||
printf '%s:base-v%s-preview-%s\n' \
|
||||
"${BASE_REPO}" "${metadata_version}" "${base_hash}"
|
||||
}
|
||||
|
||||
find_matching_base_content_ref() {
|
||||
local expected_hash="$1"
|
||||
local expected_version="$2"
|
||||
local image_ref=""
|
||||
local digest=""
|
||||
local immutable_ref=""
|
||||
local label_values=""
|
||||
local remote_hash=""
|
||||
local remote_version=""
|
||||
local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
|
||||
local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
|
||||
local attempt=0
|
||||
local inspect_status=0
|
||||
shift 2
|
||||
|
||||
if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
|
||||
|| ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
|
||||
echo "Invalid image identity retry configuration" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for image_ref in "$@"; do
|
||||
if ! digest=$(resolve_image_digest "${image_ref}"); then
|
||||
continue
|
||||
fi
|
||||
immutable_ref="${image_ref}@${digest}"
|
||||
label_values=""
|
||||
for ((attempt = 1; attempt <= attempts; attempt++)); do
|
||||
inspect_status=0
|
||||
label_values=$(docker buildx imagetools inspect "${immutable_ref}" \
|
||||
--format '{{ index .Image.Config.Labels "vllm.rocm_base.content_hash" }} {{ index .Image.Config.Labels "vllm.rocm_base.metadata_version" }}' \
|
||||
2>/dev/null) || inspect_status=$?
|
||||
remote_hash=""
|
||||
remote_version=""
|
||||
if ((inspect_status == 0)) && [[ -n "${label_values}" ]]; then
|
||||
read -r remote_hash remote_version _ <<< "${label_values}"
|
||||
if [[ "${remote_hash}" == "${expected_hash}" \
|
||||
&& "${remote_version}" == "${expected_version}" ]]; then
|
||||
printf '%s\n' "${immutable_ref}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
if ((attempt < attempts)); then
|
||||
printf \
|
||||
'ROCm base identity lookup incomplete or mismatched for %s (%d/%d); retrying\n' \
|
||||
"${immutable_ref}" "${attempt}" "${attempts}" >&2
|
||||
sleep "${delay_secs}"
|
||||
fi
|
||||
done
|
||||
if ((inspect_status == 0)) \
|
||||
&& [[ -n "${remote_hash}" && -n "${remote_version}" ]]; then
|
||||
echo "ROCm base content ref has unexpected identity: ${immutable_ref}" >&2
|
||||
echo " expected hash/version: ${expected_hash} / ${expected_version}" >&2
|
||||
echo " found hash/version: ${remote_hash} / ${remote_version}" >&2
|
||||
# A content-addressed ref with the wrong identity is corrupt. The
|
||||
# stable alias is allowed to describe older content and is only a
|
||||
# migration/recovery candidate.
|
||||
if [[ "${image_ref}" != "${BASE_REPO}:base" ]]; then
|
||||
return 2
|
||||
fi
|
||||
fi
|
||||
echo "Failed to read complete ROCm base identity labels: ${immutable_ref}" >&2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
rocm_version_from_base_image() {
|
||||
@@ -110,195 +272,51 @@ rocm_version_from_base_image() {
|
||||
tag_component "${version:-${base_image}}" 16
|
||||
}
|
||||
|
||||
git_diff_changed_base() {
|
||||
local range="$1"
|
||||
[[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]]
|
||||
}
|
||||
|
||||
short_git_ref() {
|
||||
local ref="$1"
|
||||
|
||||
git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}"
|
||||
}
|
||||
|
||||
extract_arg_default_from_ref() {
|
||||
local ref="$1"
|
||||
local arg_name="$2"
|
||||
local content=""
|
||||
|
||||
content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)"
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
<<<"${content}" | head -1
|
||||
}
|
||||
|
||||
log_arg_default_changes() {
|
||||
local old_ref="$1"
|
||||
local new_ref="$2"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local arg_name=""
|
||||
local old_value=""
|
||||
local new_value=""
|
||||
local changed=0
|
||||
|
||||
echo "Changed ROCm base ARG defaults:"
|
||||
for arg_name in ${content_args}; do
|
||||
old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")"
|
||||
new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")"
|
||||
if [[ "${old_value}" != "${new_value}" ]]; then
|
||||
echo " - ${arg_name}: ${old_value:-<unset>} -> ${new_value:-<unset>}"
|
||||
changed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${changed}" == "0" ]]; then
|
||||
echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults"
|
||||
fi
|
||||
}
|
||||
|
||||
log_arg_line_diff() {
|
||||
local range="$1"
|
||||
local arg_diff=""
|
||||
|
||||
arg_diff="$(
|
||||
git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \
|
||||
| awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \
|
||||
|| true
|
||||
)"
|
||||
|
||||
if [[ -n "${arg_diff}" ]]; then
|
||||
echo "Changed Dockerfile ARG lines:"
|
||||
printf '%s\n' "${arg_diff}"
|
||||
fi
|
||||
}
|
||||
|
||||
log_rocm_base_change_check() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
local old_short=""
|
||||
local head_short=""
|
||||
|
||||
old_short="$(short_git_ref "${old_ref}")"
|
||||
head_short="$(short_git_ref HEAD)"
|
||||
|
||||
echo "--- :mag: ROCm base refresh check"
|
||||
echo "Context: ${context}"
|
||||
echo "Dockerfile: ${DOCKERFILE}"
|
||||
echo "Base revision: ${old_short}"
|
||||
echo "Head revision: ${head_short}"
|
||||
echo "Git diff range: ${range}"
|
||||
}
|
||||
|
||||
log_rocm_base_rebuild_reason() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
local changed_files=""
|
||||
|
||||
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
|
||||
|
||||
changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)"
|
||||
echo "Changed files:"
|
||||
if [[ -n "${changed_files}" ]]; then
|
||||
sed 's/^/ - /' <<<"${changed_files}"
|
||||
else
|
||||
echo " - ${DOCKERFILE}"
|
||||
fi
|
||||
log_arg_default_changes "${old_ref}" HEAD
|
||||
log_arg_line_diff "${range}"
|
||||
echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed."
|
||||
}
|
||||
|
||||
rocm_base_changed_in_range() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
|
||||
if git_diff_changed_base "${range}"; then
|
||||
log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
|
||||
echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged."
|
||||
return 1
|
||||
}
|
||||
|
||||
rocm_base_changed() {
|
||||
local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
|
||||
local base_ref="refs/remotes/origin/${base_branch}"
|
||||
local merge_base=""
|
||||
|
||||
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
|
||||
echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then
|
||||
echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
echo "Not in a git checkout; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
|
||||
git fetch --no-tags --depth=200 origin \
|
||||
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
|
||||
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
|
||||
if [[ -z "${merge_base}" ]]; then
|
||||
echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
if rocm_base_changed_in_range \
|
||||
"pull request build against ${base_ref}" \
|
||||
"${merge_base}...HEAD" \
|
||||
"${merge_base}"; then
|
||||
return 0
|
||||
fi
|
||||
elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \
|
||||
&& git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
|
||||
if rocm_base_changed_in_range \
|
||||
"stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \
|
||||
"HEAD~1..HEAD" \
|
||||
"HEAD~1"; then
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
git fetch --no-tags --depth=200 origin \
|
||||
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
|
||||
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
|
||||
if [[ -z "${merge_base}" ]]; then
|
||||
echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
if rocm_base_changed_in_range \
|
||||
"branch build against ${base_ref}" \
|
||||
"${merge_base}...HEAD" \
|
||||
"${merge_base}"; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
should_push_stable_tag() {
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \
|
||||
&& "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]]
|
||||
is_trusted_main_build
|
||||
}
|
||||
|
||||
trusted_main_tip_matches_build() {
|
||||
local branch="${ROCM_BASE_STABLE_BRANCH:-main}"
|
||||
local build_commit="${BUILDKITE_COMMIT:-}"
|
||||
local remote_tip=""
|
||||
|
||||
is_trusted_main_build || return 1
|
||||
if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
echo "Skipping ROCm stable tag: Buildkite commit is missing or invalid" >&2
|
||||
return 1
|
||||
fi
|
||||
remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \
|
||||
"refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }')
|
||||
if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
echo "Skipping ROCm stable tag: could not resolve remote ${branch} tip" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then
|
||||
echo "Skipping ROCm stable tag: ${branch} changed after ${build_commit} (${remote_tip})" >&2
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
promote_stable_base_tag() {
|
||||
local source_ref="$1"
|
||||
local stable_tag="$2"
|
||||
|
||||
ROCM_BASE_STABLE_TAG_UPDATED=0
|
||||
should_push_stable_tag && trusted_main_tip_matches_build || return 0
|
||||
|
||||
docker buildx imagetools create --prefer-index=false \
|
||||
-t "${stable_tag}" "${source_ref}"
|
||||
ROCM_BASE_STABLE_TAG_UPDATED=1
|
||||
}
|
||||
|
||||
setup_builder() {
|
||||
@@ -315,29 +333,43 @@ compute_base_content_hash() {
|
||||
local use_sccache="$1"
|
||||
local base_image_digest="$2"
|
||||
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local sccache_arg=""
|
||||
local sccache_value=""
|
||||
local -a content_paths=()
|
||||
local -a content_arg_names=()
|
||||
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
read -r -a content_arg_names <<< "${content_args}"
|
||||
|
||||
{
|
||||
printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
|
||||
printf 'dockerfile:%s\n' "${DOCKERFILE}"
|
||||
printf 'resolved-build-args:\n'
|
||||
hash_rocm_base_arg_values \
|
||||
"${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}"
|
||||
printf 'base-image-digest:%s\n' "${base_image_digest}"
|
||||
printf 'use-sccache:%s\n' "${use_sccache}"
|
||||
if [[ "${use_sccache}" == "1" ]]; then
|
||||
# These values can change the installed binary or final image
|
||||
# configuration. SCCACHE_ENDPOINT is transport-only and therefore
|
||||
# deliberately excluded from image identity.
|
||||
for sccache_arg in \
|
||||
SCCACHE_DOWNLOAD_URL \
|
||||
SCCACHE_BUCKET_NAME \
|
||||
SCCACHE_REGION_NAME \
|
||||
SCCACHE_S3_NO_CREDENTIALS; do
|
||||
sccache_value="${!sccache_arg:-}"
|
||||
if [[ -z "${sccache_value}" ]]; then
|
||||
sccache_value=$(extract_arg_default "${sccache_arg}")
|
||||
fi
|
||||
printf 'arg:%s=%s\n' \
|
||||
"${sccache_arg}" "${sccache_value:-<empty>}"
|
||||
done
|
||||
fi
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
build_base_image() {
|
||||
local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}"
|
||||
local base_hash=""
|
||||
local build_date=""
|
||||
local build_suffix=""
|
||||
local base_image_arg=""
|
||||
local base_image_digest=""
|
||||
local pinned_base_image=""
|
||||
local rocm_version=""
|
||||
local triton_arg=""
|
||||
local pytorch_arg=""
|
||||
@@ -348,20 +380,19 @@ build_base_image() {
|
||||
local mori_arg=""
|
||||
local python_version_arg=""
|
||||
local pytorch_rocm_arch_arg=""
|
||||
local pytorch_branch=""
|
||||
local aiter_branch=""
|
||||
local dependency_summary=""
|
||||
local descriptor=""
|
||||
local ci_descriptor=""
|
||||
local descriptive_tag=""
|
||||
local stable_tag="${BASE_REPO}:base"
|
||||
local ci_descriptive_tag=""
|
||||
local trusted_content_tag=""
|
||||
local scoped_content_tag=""
|
||||
local writable_content_tag=""
|
||||
local build_ref=""
|
||||
local immutable_ref=""
|
||||
local selected_digest=""
|
||||
local stable_digest=""
|
||||
local reuse_status=0
|
||||
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local content_files_hash=""
|
||||
local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}"
|
||||
local -a tags=()
|
||||
local -a no_cache_args=()
|
||||
local -a sccache_args=()
|
||||
local -a content_paths=()
|
||||
|
||||
@@ -369,13 +400,21 @@ build_base_image() {
|
||||
echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}"
|
||||
if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
|
||||
build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}"
|
||||
if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then
|
||||
echo "Invalid ROCm base metadata version: ${metadata_version}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "${use_sccache}" != "0" && "${use_sccache}" != "1" ]]; then
|
||||
echo "ROCm base USE_SCCACHE must be 0 or 1: ${use_sccache}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
base_image_arg="$(extract_arg_default BASE_IMAGE)"
|
||||
base_image_digest="$(resolve_image_digest "${base_image_arg}")"
|
||||
if ! base_image_digest=$(resolve_image_digest "${base_image_arg}"); then
|
||||
echo "Failed to resolve ROCm base input image: ${base_image_arg}" >&2
|
||||
return 1
|
||||
fi
|
||||
pinned_base_image="${base_image_arg%@*}@${base_image_digest}"
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
|
||||
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
|
||||
@@ -389,125 +428,161 @@ build_base_image() {
|
||||
mori_arg="$(extract_arg_default MORI_BRANCH)"
|
||||
python_version_arg="$(extract_arg_default PYTHON_VERSION)"
|
||||
pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)"
|
||||
pytorch_branch="$(tag_component "${pytorch_arg}" 16)"
|
||||
aiter_branch="$(tag_component "${aiter_arg}" 24)"
|
||||
dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}"
|
||||
descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
|
||||
ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
|
||||
|
||||
descriptive_tag="${BASE_REPO}:${descriptor}"
|
||||
ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}"
|
||||
|
||||
tags=(-t "${descriptive_tag}")
|
||||
if should_push_stable_tag; then
|
||||
tags+=(-t "${stable_tag}")
|
||||
metadata_set "rocm-base-push-stable-tag" "1"
|
||||
trusted_content_tag=$(trusted_base_content_ref "${base_hash}" "${metadata_version}")
|
||||
scoped_content_tag=$(scoped_base_content_ref "${base_hash}" "${metadata_version}")
|
||||
# Preview writes share an exact-content namespace so stacked PRs can reuse
|
||||
# identical images. They may import the trusted ref, but never overwrite it.
|
||||
if is_trusted_main_build; then
|
||||
scoped_content_tag="${trusted_content_tag}"
|
||||
writable_content_tag="${trusted_content_tag}"
|
||||
else
|
||||
metadata_set "rocm-base-push-stable-tag" "0"
|
||||
writable_content_tag="${scoped_content_tag}"
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then
|
||||
no_cache_args=(--no-cache)
|
||||
configure_rocm_base_layer_cache
|
||||
|
||||
if [[ "${use_sccache}" == "1" ]]; then
|
||||
for env_name in \
|
||||
SCCACHE_DOWNLOAD_URL \
|
||||
SCCACHE_ENDPOINT \
|
||||
SCCACHE_BUCKET_NAME \
|
||||
SCCACHE_REGION_NAME \
|
||||
SCCACHE_S3_NO_CREDENTIALS; do
|
||||
if [[ -n "${!env_name:-}" ]]; then
|
||||
sccache_args+=(--build-arg "${env_name}=${!env_name}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
for env_name in \
|
||||
SCCACHE_DOWNLOAD_URL \
|
||||
SCCACHE_ENDPOINT \
|
||||
SCCACHE_BUCKET_NAME \
|
||||
SCCACHE_REGION_NAME \
|
||||
SCCACHE_S3_NO_CREDENTIALS; do
|
||||
if [[ -n "${!env_name:-}" ]]; then
|
||||
sccache_args+=(--build-arg "${env_name}=${!env_name}")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- :docker: Building ROCm base image"
|
||||
echo "--- :docker: Preparing ROCm base image"
|
||||
echo "Dockerfile: ${DOCKERFILE}"
|
||||
echo "Descriptive tag: ${descriptive_tag}"
|
||||
echo "Trusted content tag: ${trusted_content_tag}"
|
||||
echo "Writable content tag: ${writable_content_tag}"
|
||||
echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))"
|
||||
echo "Content hash: ${base_hash}"
|
||||
echo "Dependency summary: ${dependency_summary}"
|
||||
echo "USE_SCCACHE: ${use_sccache}"
|
||||
echo "BuildKit layer cache: ${ROCM_BASE_LAYER_CACHE_REF}"
|
||||
|
||||
docker buildx build \
|
||||
"${no_cache_args[@]}" \
|
||||
--pull \
|
||||
--progress "${BUILDKIT_PROGRESS:-plain}" \
|
||||
--file "${DOCKERFILE}" \
|
||||
--build-arg "USE_SCCACHE=${use_sccache}" \
|
||||
"${sccache_args[@]}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
|
||||
--label "org.opencontainers.image.vendor=vLLM" \
|
||||
--label "org.opencontainers.image.title=vLLM ROCm base" \
|
||||
--label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \
|
||||
--label "vllm.rocm_base.metadata_version=${metadata_version}" \
|
||||
--label "vllm.rocm_base.content_hash=${base_hash}" \
|
||||
--label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
|
||||
--label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
|
||||
--label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \
|
||||
--label "vllm.rocm_base.image.stable=${stable_tag}" \
|
||||
--label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \
|
||||
--label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \
|
||||
--label "vllm.rocm_base.descriptor=${descriptor}" \
|
||||
--label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
|
||||
--label "vllm.rocm_base.base_image=${base_image_arg}" \
|
||||
--label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
|
||||
--label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
|
||||
--label "vllm.rocm_base.dependency.python=${python_version_arg}" \
|
||||
--label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
|
||||
--label "vllm.rocm_base.dependency.triton=${triton_arg}" \
|
||||
--label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
|
||||
--label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
|
||||
--label "vllm.rocm_base.dependency.mori=${mori_arg}" \
|
||||
--label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
|
||||
"${tags[@]}" \
|
||||
--push \
|
||||
.
|
||||
if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" != "1" \
|
||||
&& "${ROCM_BASE_NO_CACHE:-0}" != "1" ]]; then
|
||||
reuse_status=0
|
||||
if [[ "${trusted_content_tag}" == "${scoped_content_tag}" ]]; then
|
||||
immutable_ref=$(find_matching_base_content_ref \
|
||||
"${base_hash}" "${metadata_version}" \
|
||||
"${trusted_content_tag}" "${stable_tag}") || reuse_status=$?
|
||||
else
|
||||
immutable_ref=$(find_matching_base_content_ref \
|
||||
"${base_hash}" "${metadata_version}" \
|
||||
"${trusted_content_tag}" "${scoped_content_tag}" \
|
||||
"${stable_tag}") \
|
||||
|| reuse_status=$?
|
||||
fi
|
||||
else
|
||||
reuse_status=1
|
||||
echo "Forced/no-cache ROCm base build; bypassing content image reuse"
|
||||
fi
|
||||
|
||||
docker buildx imagetools inspect "${descriptive_tag}" >/dev/null
|
||||
if ((reuse_status > 1)); then
|
||||
return "${reuse_status}"
|
||||
fi
|
||||
|
||||
if [[ ${reuse_status} -eq 0 && -n "${immutable_ref}" ]]; then
|
||||
echo "Reusing ROCm base image with matching content: ${immutable_ref}"
|
||||
if [[ "${immutable_ref%@*}" == "${stable_tag}" \
|
||||
&& "${writable_content_tag}" != "${stable_tag}" ]]; then
|
||||
echo "Backfilling ROCm base content tag: ${writable_content_tag}"
|
||||
docker buildx imagetools create --prefer-index=false \
|
||||
-t "${writable_content_tag}" "${immutable_ref}"
|
||||
if ! immutable_ref=$(find_matching_base_content_ref \
|
||||
"${base_hash}" "${metadata_version}" \
|
||||
"${writable_content_tag}"); then
|
||||
echo "Backfilled ROCm base content tag failed identity validation" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
build_ref="${writable_content_tag}"
|
||||
echo "No reusable ROCm base content image found; building ${build_ref}"
|
||||
setup_builder
|
||||
docker buildx build \
|
||||
"${ROCM_BASE_CACHE_ARGS[@]}" \
|
||||
--pull \
|
||||
--provenance=false \
|
||||
--progress "${BUILDKIT_PROGRESS:-plain}" \
|
||||
--file "${DOCKERFILE}" \
|
||||
--build-arg "BASE_IMAGE=${pinned_base_image}" \
|
||||
--build-arg "USE_SCCACHE=${use_sccache}" \
|
||||
"${sccache_args[@]}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
|
||||
--label "org.opencontainers.image.vendor=vLLM" \
|
||||
--label "org.opencontainers.image.title=vLLM ROCm base" \
|
||||
--label "vllm.rocm_base.metadata_version=${metadata_version}" \
|
||||
--label "vllm.rocm_base.content_hash=${base_hash}" \
|
||||
--label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
|
||||
--label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
|
||||
--label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
|
||||
--label "vllm.rocm_base.base_image=${pinned_base_image}" \
|
||||
--label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
|
||||
--label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
|
||||
--label "vllm.rocm_base.dependency.python=${python_version_arg}" \
|
||||
--label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
|
||||
--label "vllm.rocm_base.dependency.triton=${triton_arg}" \
|
||||
--label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
|
||||
--label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
|
||||
--label "vllm.rocm_base.dependency.mori=${mori_arg}" \
|
||||
--label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
|
||||
-t "${build_ref}" \
|
||||
--push \
|
||||
.
|
||||
if ! immutable_ref=$(find_matching_base_content_ref \
|
||||
"${base_hash}" "${metadata_version}" "${build_ref}"); then
|
||||
echo "Published ROCm base image failed identity validation: ${build_ref}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
selected_digest="${immutable_ref##*@}"
|
||||
metadata_set "rocm-base-image" "${immutable_ref}"
|
||||
metadata_set "rocm-base-push-stable-tag" "0"
|
||||
stable_digest=$(resolve_image_digest "${stable_tag}" 2>/dev/null || true)
|
||||
if [[ -n "${stable_digest}" && "${stable_digest}" == "${selected_digest}" ]]; then
|
||||
echo "--- :white_check_mark: Existing ROCm base is current"
|
||||
echo "Stable ${stable_tag} already resolves to ${selected_digest}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
promote_stable_base_tag "${immutable_ref}" "${stable_tag}"
|
||||
|
||||
metadata_set "rocm-base-push-stable-tag" "${ROCM_BASE_STABLE_TAG_UPDATED}"
|
||||
metadata_set "rocm-base-refresh" "1"
|
||||
metadata_set "rocm-base-image" "${descriptive_tag}"
|
||||
metadata_set "rocm-base-image-descriptive" "${descriptive_tag}"
|
||||
metadata_set "rocm-base-image-stable" "${stable_tag}"
|
||||
metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}"
|
||||
metadata_set "rocm-base-metadata-version" "${metadata_version}"
|
||||
metadata_set "rocm-base-content-hash" "${base_hash}"
|
||||
metadata_set "rocm-base-content-files-hash" "${content_files_hash}"
|
||||
metadata_set "rocm-base-content-files" "${content_files}"
|
||||
metadata_set "rocm-base-content-args" "${content_args}"
|
||||
metadata_set "rocm-base-base-image-digest" "${base_image_digest}"
|
||||
metadata_set "rocm-base-dockerfile" "${DOCKERFILE}"
|
||||
metadata_set "rocm-base-descriptor" "${descriptor}"
|
||||
metadata_set "rocm-base-dependency-summary" "${dependency_summary}"
|
||||
metadata_set "rocm-base-dependency-rocm" "${rocm_version}"
|
||||
metadata_set "rocm-base-dependency-python" "${python_version_arg}"
|
||||
metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}"
|
||||
metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}"
|
||||
metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}"
|
||||
metadata_set "rocm-base-dependency-triton" "${triton_arg}"
|
||||
metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}"
|
||||
metadata_set "rocm-base-dependency-aiter" "${aiter_arg}"
|
||||
metadata_set "rocm-base-dependency-mori" "${mori_arg}"
|
||||
metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}"
|
||||
metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}"
|
||||
|
||||
echo "--- :white_check_mark: ROCm base image published"
|
||||
echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds"
|
||||
echo "Use BASE_IMAGE=${immutable_ref} for downstream ROCm CI builds"
|
||||
}
|
||||
|
||||
main() {
|
||||
metadata_set "rocm-base-refresh" "0"
|
||||
|
||||
if ! rocm_base_changed; then
|
||||
echo "ROCm base Dockerfile did not change; skipping base image refresh"
|
||||
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
|
||||
local stable_ref="${BASE_REPO}:base"
|
||||
local stable_digest=""
|
||||
|
||||
if ! stable_digest=$(resolve_image_digest "${stable_ref}"); then
|
||||
echo "Could not pin explicitly skipped ROCm base: ${stable_ref}" >&2
|
||||
return 1
|
||||
fi
|
||||
metadata_set "rocm-base-image" "${stable_ref}@${stable_digest}"
|
||||
metadata_set "rocm-base-push-stable-tag" "0"
|
||||
echo "ROCM_BASE_REFRESH_SKIP=1; pinned current stable base at ${stable_digest}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
setup_builder
|
||||
build_base_image
|
||||
}
|
||||
|
||||
main "$@"
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
|
||||
@@ -3,7 +3,18 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}"
|
||||
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
|
||||
base_refreshed=""
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
base_refreshed="$(buildkite-agent meta-data get rocm-base-refresh 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ "${base_refreshed}" != "1" ]]; then
|
||||
echo "ROCM_CI_ARTIFACT_ONLY=1; no full image was built, skipping smoke test"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
|
||||
|
||||
docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
|
||||
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
/.venv
|
||||
/build
|
||||
/artifacts/vllm-rocm-install/
|
||||
/bake-config-build-*.json
|
||||
/wheel-export/
|
||||
dist
|
||||
vllm/*.so
|
||||
vllm/vllm-rs
|
||||
|
||||
+25
-18
@@ -253,21 +253,32 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
|
||||
sccache --show-stats; \
|
||||
fi
|
||||
|
||||
# -----------------------
|
||||
# Shared vLLM build dependencies
|
||||
#
|
||||
# Install requirements once, before bringing source-specific inputs into either
|
||||
# consumer. Native compilation and wheel packaging then share this expensive
|
||||
# layer while retaining their independent source cache boundaries.
|
||||
FROM base AS build_vllm_dependencies
|
||||
ARG COMMON_WORKDIR
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY --from=fetch_vllm \
|
||||
${COMMON_WORKDIR}/vllm/requirements/common.txt \
|
||||
${COMMON_WORKDIR}/vllm/requirements/rocm.txt \
|
||||
requirements/
|
||||
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
# -----------------------
|
||||
# vLLM native build stages
|
||||
#
|
||||
# csrc-build intentionally copies only files that affect ROCm native extension
|
||||
# compilation. That keeps unrelated CI/test/docs edits from invalidating the
|
||||
# expensive HIP/C++ build layer.
|
||||
FROM base AS csrc-build
|
||||
ARG COMMON_WORKDIR
|
||||
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||
|
||||
COPY requirements/rocm.txt requirements/rocm.txt
|
||||
COPY requirements/common.txt requirements/common.txt
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements/rocm.txt
|
||||
|
||||
FROM build_vllm_dependencies AS csrc-build
|
||||
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
|
||||
# but it still participates in the BuildKit cache key for this RUN step.
|
||||
COPY setup.py CMakeLists.txt ./
|
||||
@@ -277,7 +288,6 @@ COPY csrc csrc/
|
||||
COPY vllm/envs.py vllm/envs.py
|
||||
COPY vllm/__init__.py vllm/__init__.py
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
|
||||
|
||||
RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
|
||||
@@ -296,11 +306,10 @@ RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproj
|
||||
|
||||
# Build the full vLLM ROCm wheel by reusing the native extension wheel from
|
||||
# csrc-build. This stage still rebuilds for Python/package changes, but skips
|
||||
# the expensive HIP/C++ compile when native inputs are unchanged.
|
||||
FROM fetch_vllm AS build_vllm
|
||||
ARG COMMON_WORKDIR
|
||||
ENV VLLM_TARGET_DEVICE=rocm
|
||||
|
||||
# the expensive dependency installation and HIP/C++ compilation when their
|
||||
# respective inputs are unchanged.
|
||||
FROM build_vllm_dependencies AS build_vllm
|
||||
COPY --from=fetch_vllm ${COMMON_WORKDIR}/vllm ${COMMON_WORKDIR}/vllm
|
||||
COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
|
||||
|
||||
# Drop the pre-built Rust artifacts into the source tree. setup.py detects
|
||||
@@ -308,9 +317,7 @@ COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
|
||||
COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
|
||||
COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/
|
||||
|
||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||
cd vllm \
|
||||
&& uv pip install --system -r requirements/rocm.txt \
|
||||
RUN cd "${COMMON_WORKDIR}/vllm" \
|
||||
&& export VLLM_USE_PRECOMPILED=1 \
|
||||
&& export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
|
||||
&& export VLLM_DOCKER_BUILD_CONTEXT=1 \
|
||||
|
||||
+47
-16
@@ -134,11 +134,7 @@ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
|
||||
###
|
||||
FROM base AS build_pytorch
|
||||
ARG PYTORCH_BRANCH
|
||||
ARG PYTORCH_VISION_BRANCH
|
||||
ARG PYTORCH_AUDIO_BRANCH
|
||||
ARG PYTORCH_REPO
|
||||
ARG PYTORCH_VISION_REPO
|
||||
ARG PYTORCH_AUDIO_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN apt-get update && apt-get install -y pkg-config liblzma-dev
|
||||
@@ -156,8 +152,27 @@ RUN cd pytorch && python3 tools/amd_build/build_amd.py \
|
||||
&& sccache --show-stats; \
|
||||
fi \
|
||||
&& CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
### Shared PyTorch runtime for dependent builds
|
||||
###
|
||||
FROM base AS build_pytorch_runtime
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \
|
||||
pip install /install/*.whl
|
||||
|
||||
|
||||
###
|
||||
### TorchVision build
|
||||
###
|
||||
FROM build_pytorch_runtime AS build_torchvision
|
||||
ARG PYTORCH_VISION_BRANCH
|
||||
ARG PYTORCH_VISION_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN git clone ${PYTORCH_VISION_REPO} vision
|
||||
RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
@@ -166,23 +181,31 @@ RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
|
||||
&& export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
|
||||
fi \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
### TorchAudio build
|
||||
###
|
||||
FROM build_pytorch_runtime AS build_torchaudio
|
||||
ARG PYTORCH_AUDIO_BRANCH
|
||||
ARG PYTORCH_AUDIO_REPO
|
||||
ARG USE_SCCACHE
|
||||
|
||||
RUN git clone ${PYTORCH_AUDIO_REPO} audio
|
||||
RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
|
||||
&& git submodule update --init --recursive \
|
||||
&& pip install -r requirements.txt \
|
||||
&& git submodule update --init --recursive
|
||||
RUN cd audio && pip install -r requirements.txt
|
||||
RUN cd audio \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
||||
export HIP_CLANG_PATH=/opt/sccache-wrappers \
|
||||
&& export CMAKE_C_COMPILER_LAUNCHER=sccache \
|
||||
&& export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
|
||||
fi \
|
||||
&& python3 setup.py bdist_wheel --dist-dir=dist \
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
|
||||
&& pip install dist/*.whl
|
||||
RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
|
||||
&& cp /app/vision/dist/*.whl /app/install \
|
||||
&& cp /app/audio/dist/*.whl /app/install
|
||||
&& if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
|
||||
RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install
|
||||
|
||||
|
||||
###
|
||||
@@ -263,6 +286,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
|
||||
@@ -277,6 +304,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
|
||||
cp /install/*.whl /app/debs
|
||||
RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
|
||||
@@ -315,4 +346,4 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
|
||||
&& echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
|
||||
&& echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
|
||||
&& echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
|
||||
&& echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
|
||||
|
||||
+15
-24
@@ -338,33 +338,22 @@ group "test-rocm-ci-with-artifacts" {
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed
|
||||
# per-commit image is useful.
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed,
|
||||
# build-scoped image is useful.
|
||||
group "test-rocm-ci-with-wheel" {
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||
}
|
||||
|
||||
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
|
||||
# to the primary tag for this build. Builds always publish a content-scoped tag
|
||||
# when the ci_base content hash is available. Builds with BUILDKITE_COMMIT also
|
||||
# publish a commit-scoped tag, either as the primary tag or an additional alias.
|
||||
# NIGHTLY=1 builds on the stable branch can additionally set
|
||||
# CI_BASE_IMAGE_TAG_STABLE to refresh rocm/vllm-dev:ci_base.
|
||||
# Primary output tag for the ci_base build. In Buildkite this is a unique,
|
||||
# build-scoped tag; ci-bake-rocm.sh validates it before creating content and
|
||||
# stable aliases.
|
||||
variable "CI_BASE_IMAGE_TAG" {
|
||||
default = "rocm/vllm-dev:ci_base"
|
||||
}
|
||||
|
||||
# Supplemental tags only. ci-bake-rocm.sh leaves these empty when the same ref
|
||||
# is already the primary CI_BASE_IMAGE_TAG.
|
||||
variable "CI_BASE_IMAGE_TAG_COMMIT_EXTRA" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "CI_BASE_IMAGE_TAG_CONTENT_EXTRA" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "CI_BASE_IMAGE_TAG_STABLE" {
|
||||
# Versioned, content-addressed trusted ref. Preview builds import it read-only;
|
||||
# the wrapper creates their source-scoped content alias after validation.
|
||||
variable "CI_BASE_TRUSTED_CONTENT_REF" {
|
||||
default = ""
|
||||
}
|
||||
|
||||
@@ -403,21 +392,23 @@ target "deepep-rocm-ci" {
|
||||
# Uses inline cache metadata on the ci_base image itself instead of exporting a
|
||||
# separate registry cache artifact.
|
||||
target "ci-base-rocm-ci" {
|
||||
inherits = ["_common-rocm", "_ci-rocm", "_labels"]
|
||||
inherits = ["_common-rocm"]
|
||||
target = "ci_base"
|
||||
args = {
|
||||
max_jobs = CI_MAX_JOBS
|
||||
}
|
||||
cache-from = concat(
|
||||
compact([
|
||||
CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "",
|
||||
CI_BASE_IMAGE_TAG_COMMIT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_COMMIT_EXTRA}" : "",
|
||||
CI_BASE_IMAGE_TAG_CONTENT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_CONTENT_EXTRA}" : "",
|
||||
CI_BASE_IMAGE_TAG_STABLE != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_STABLE}" : "",
|
||||
CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "",
|
||||
]),
|
||||
# Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages
|
||||
# are cache hits even when ci_base itself needs rebuilding.
|
||||
get_cache_from_rocm_deps(),
|
||||
)
|
||||
cache-to = ["type=inline"]
|
||||
tags = compact([CI_BASE_IMAGE_TAG, CI_BASE_IMAGE_TAG_COMMIT_EXTRA, CI_BASE_IMAGE_TAG_CONTENT_EXTRA, CI_BASE_IMAGE_TAG_STABLE])
|
||||
tags = compact([CI_BASE_IMAGE_TAG])
|
||||
attest = ["type=provenance,disabled=true"]
|
||||
output = ["type=registry"]
|
||||
}
|
||||
|
||||
|
||||
@@ -171,10 +171,6 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None:
|
||||
"requirements/common.txt",
|
||||
"requirements/rocm.txt",
|
||||
"requirements/test/rocm.txt",
|
||||
"docker/Dockerfile.rocm_base",
|
||||
"docker/Dockerfile.rocm",
|
||||
"docker/ci-rocm.hcl",
|
||||
"docker/docker-bake-rocm.hcl",
|
||||
".buildkite/scripts/ci-bake-rocm.sh",
|
||||
):
|
||||
assert expected in ci_bake
|
||||
|
||||
Reference in New Issue
Block a user