[CI][ROCm] Avoid redundant image pulls during smoke validation (#54408)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-31 21:48:25 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent c28feab989
commit 907b1a7f22
5 changed files with 277 additions and 31 deletions
+54 -5
View File
@@ -842,6 +842,29 @@ should_upload_wheel_artifacts() {
|| "${TARGET}" == *"artifact"* ]]
}
should_export_rocm_smoke() {
[[ "${TARGET}" == "test-rocm-ci-with-wheel" \
|| "${TARGET}" == "smoke-test-rocm-ci" ]]
}
verify_rocm_smoke_export() {
local marker="./build/rocm-smoke-export/vllm-smoke-ok"
local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
local actual_smoke_id=""
should_export_rocm_smoke || return 0
if [[ ! -f "${marker}" ]]; then
echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2
return 1
fi
actual_smoke_id="$(< "${marker}")"
if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
>&2
return 1
fi
}
get_remote_image_label() {
local image_ref="$1"
local label_key="$2"
@@ -1386,8 +1409,9 @@ maybe_skip_existing_image() {
echo "FORCE_BUILD=1 set; skipping existing-image check"
return 0
fi
if ! is_ci_base_target && should_upload_wheel_artifacts; then
echo "Artifact-producing targets always run for the current build"
if ! is_ci_base_target \
&& { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then
echo "Local-output targets always run for the current build"
return 0
fi
@@ -1741,7 +1765,12 @@ EOF
uses_rocm_csrc_cache() {
case "${TARGET}" in
csrc-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
csrc-rocm-ci \
| test-rocm-ci \
| test-rocm-ci-with-wheel \
| test-rocm-ci-with-artifacts \
| export-wheel-rocm \
| smoke-test-rocm-ci)
return 0
;;
*)
@@ -1752,7 +1781,12 @@ uses_rocm_csrc_cache() {
uses_rocm_rust_cache() {
case "${TARGET}" in
rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
rust-rocm-ci \
| test-rocm-ci \
| test-rocm-ci-with-wheel \
| test-rocm-ci-with-artifacts \
| export-wheel-rocm \
| smoke-test-rocm-ci)
return 0
;;
*)
@@ -2140,7 +2174,7 @@ write_rocm_cache_override() {
# exporter unless it is requested explicitly.
if ((${#rust_cache_to[@]} > 0)); then
case "${TARGET}" in
test-rocm-ci|export-wheel-rocm)
test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci)
BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
;;
esac
@@ -2190,6 +2224,15 @@ EOF
cat <<EOF
}
target "smoke-test-rocm-ci" {
cache-from = concat(
get_cache_from_rocm(),
EOF
write_hcl_string_list " " "${combined_content_cache_from[@]}"
cat <<EOF
)
}
target "export-wheel-rocm" {
cache-from = concat(
get_cache_from_rocm(),
@@ -2675,8 +2718,14 @@ main() {
# clean prevents a failed/retried export from packaging a stale wheel.
rm -rf ./wheel-export
fi
if should_export_rocm_smoke; then
# The marker is a build output, not a cache. Never accept stale output
# from an earlier build or retry.
rm -rf ./build/rocm-smoke-export
fi
seed_dependency_caches_if_needed
run_bake
verify_rocm_smoke_export
promote_stable_ci_base_tag
publish_ci_base_handoff_ref
upload_wheel_artifacts_if_present
+66 -25
View File
@@ -3,6 +3,53 @@
set -euo pipefail
run_smoke_checks() {
local required_dir=""
for required_dir in \
/vllm-workspace \
/vllm-workspace/tests \
/vllm-workspace/src/vllm; do
if [[ ! -d "${required_dir}" ]]; then
echo "Missing directory: ${required_dir}" >&2
return 1
fi
done
if [[ ! -x /vllm-workspace/src/vllm/vllm-rs ]]; then
echo "Missing executable: /vllm-workspace/src/vllm/vllm-rs" >&2
return 1
fi
command -v python3
command -v uv
command -v pytest
if ! command -v amd-smi >/dev/null 2>&1 \
&& ! command -v rocminfo >/dev/null 2>&1; then
echo "No ROCm CLI found in image" >&2
return 1
fi
PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY'
import torch
import vllm
print(torch.__version__)
print(vllm.__version__)
PY
echo "AMD image smoke OK"
}
if [[ "${1:-}" == "--inside" ]]; then
run_smoke_checks
exit
fi
if (($#)); then
echo "Usage: $0 [--inside]" >&2
exit 2
fi
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
base_refreshed=""
if command -v buildkite-agent >/dev/null 2>&1; then
@@ -14,30 +61,24 @@ if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
fi
fi
smoke_marker="./build/rocm-smoke-export/vllm-smoke-ok"
expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
if [[ -f "${smoke_marker}" \
&& ( -z "${VLLM_CI_SMOKE_IMAGE:-}" \
|| "${VLLM_CI_SMOKE_IMAGE}" == "${IMAGE_TAG:-}" ) ]]; then
actual_smoke_id="$(< "${smoke_marker}")"
if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
echo "ROCm smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
>&2
exit 1
fi
echo "AMD image smoke OK (verified inside BuildKit)"
rm -f -- "${smoke_marker}"
rmdir -- "$(dirname "${smoke_marker}")" 2>/dev/null || true
exit
fi
image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
command -v python3
command -v uv
command -v pytest
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
echo No ROCm CLI found in image >&2
exit 1
fi
python3 - <<PY
import torch
import vllm
print(torch.__version__)
print(vllm.__version__)
PY
echo AMD image smoke OK
'
docker run --rm -i --network=none --entrypoint /bin/bash "${image_ref}" \
-s -- --inside < "${BASH_SOURCE[0]}"