From 907b1a7f22509b0095f2711cd47637c6ca345325 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 31 Aug 2026 19:48:25 -0700 Subject: [PATCH] [CI][ROCm] Avoid redundant image pulls during smoke validation (#54408) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/scripts/ci-bake-rocm.sh | 59 +++++++- .buildkite/scripts/rocm/smoke-test-image.sh | 91 ++++++++---- docker/Dockerfile.rocm | 11 ++ docker/ci-rocm.hcl | 18 ++- .../tools/test_docker_build_metadata_args.py | 129 ++++++++++++++++++ 5 files changed, 277 insertions(+), 31 deletions(-) diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index ef38b8be58..15f8274fc6 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -842,6 +842,29 @@ should_upload_wheel_artifacts() { || "${TARGET}" == *"artifact"* ]] } +should_export_rocm_smoke() { + [[ "${TARGET}" == "test-rocm-ci-with-wheel" \ + || "${TARGET}" == "smoke-test-rocm-ci" ]] +} + +verify_rocm_smoke_export() { + local marker="./build/rocm-smoke-export/vllm-smoke-ok" + local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}" + local actual_smoke_id="" + + should_export_rocm_smoke || return 0 + if [[ ! -f "${marker}" ]]; then + echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2 + return 1 + fi + actual_smoke_id="$(< "${marker}")" + if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then + echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \ + >&2 + return 1 + fi +} + get_remote_image_label() { local image_ref="$1" local label_key="$2" @@ -1386,8 +1409,9 @@ maybe_skip_existing_image() { echo "FORCE_BUILD=1 set; skipping existing-image check" return 0 fi - if ! is_ci_base_target && should_upload_wheel_artifacts; then - echo "Artifact-producing targets always run for the current build" + if ! is_ci_base_target \ + && { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then + echo "Local-output targets always run for the current build" return 0 fi @@ -1741,7 +1765,12 @@ EOF uses_rocm_csrc_cache() { case "${TARGET}" in - csrc-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm) + csrc-rocm-ci \ + | test-rocm-ci \ + | test-rocm-ci-with-wheel \ + | test-rocm-ci-with-artifacts \ + | export-wheel-rocm \ + | smoke-test-rocm-ci) return 0 ;; *) @@ -1752,7 +1781,12 @@ uses_rocm_csrc_cache() { uses_rocm_rust_cache() { case "${TARGET}" in - rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm) + rust-rocm-ci \ + | test-rocm-ci \ + | test-rocm-ci-with-wheel \ + | test-rocm-ci-with-artifacts \ + | export-wheel-rocm \ + | smoke-test-rocm-ci) return 0 ;; *) @@ -2140,7 +2174,7 @@ write_rocm_cache_override() { # exporter unless it is requested explicitly. if ((${#rust_cache_to[@]} > 0)); then case "${TARGET}" in - test-rocm-ci|export-wheel-rocm) + test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci) BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}") ;; esac @@ -2190,6 +2224,15 @@ EOF cat <&2 + return 1 + fi + done + if [[ ! -x /vllm-workspace/src/vllm/vllm-rs ]]; then + echo "Missing executable: /vllm-workspace/src/vllm/vllm-rs" >&2 + return 1 + fi + + command -v python3 + command -v uv + command -v pytest + + if ! command -v amd-smi >/dev/null 2>&1 \ + && ! command -v rocminfo >/dev/null 2>&1; then + echo "No ROCm CLI found in image" >&2 + return 1 + fi + + PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY' +import torch +import vllm + +print(torch.__version__) +print(vllm.__version__) +PY + + echo "AMD image smoke OK" +} + +if [[ "${1:-}" == "--inside" ]]; then + run_smoke_checks + exit +fi +if (($#)); then + echo "Usage: $0 [--inside]" >&2 + exit 2 +fi + if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then base_refreshed="" if command -v buildkite-agent >/dev/null 2>&1; then @@ -14,30 +61,24 @@ if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then fi fi +smoke_marker="./build/rocm-smoke-export/vllm-smoke-ok" +expected_smoke_id="${BUILDKITE_BUILD_ID:-local}" +if [[ -f "${smoke_marker}" \ + && ( -z "${VLLM_CI_SMOKE_IMAGE:-}" \ + || "${VLLM_CI_SMOKE_IMAGE}" == "${IMAGE_TAG:-}" ) ]]; then + actual_smoke_id="$(< "${smoke_marker}")" + if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then + echo "ROCm smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \ + >&2 + exit 1 + fi + echo "AMD image smoke OK (verified inside BuildKit)" + rm -f -- "${smoke_marker}" + rmdir -- "$(dirname "${smoke_marker}")" 2>/dev/null || true + exit +fi + image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}" -docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec ' - if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi - if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi - if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi - if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi - - command -v python3 - command -v uv - command -v pytest - - if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then - echo No ROCm CLI found in image >&2 - exit 1 - fi - - python3 - <CPU syncs in execute_model/sample_tokens during tests. ENV VLLM_GPU_SYNC_CHECK=error +# Run the structural smoke test inside BuildKit, where the test image layers +# are already available. Export only a build-scoped success marker. +FROM test AS test_smoke +ARG ROCM_SMOKE_ID +RUN --network=none \ + bash /vllm-workspace/.buildkite/scripts/rocm/smoke-test-image.sh --inside \ + && printf '%s\n' "${ROCM_SMOKE_ID:-local}" > /vllm-smoke-ok + +FROM scratch AS export_test_smoke +COPY --from=test_smoke /vllm-smoke-ok / + # ----------------------- # Final vLLM image FROM mori_base AS final_common diff --git a/docker/ci-rocm.hcl b/docker/ci-rocm.hcl index 0bdf4d1a49..1794b8468a 100644 --- a/docker/ci-rocm.hcl +++ b/docker/ci-rocm.hcl @@ -281,6 +281,7 @@ target "_ci-rocm" { args = { ARG_PYTORCH_ROCM_ARCH = PYTORCH_ROCM_ARCH CI_BASE_IMAGE = CI_BASE_IMAGE + ROCM_SMOKE_ID = BUILDKITE_BUILD_ID max_jobs = CI_MAX_JOBS } } @@ -297,6 +298,15 @@ target "test-rocm-ci" { output = ["type=registry"] } +# Validate the test image in the shared BuildKit graph and export only the +# success marker. This avoids pulling the multi-GB image into the host daemon. +target "smoke-test-rocm-ci" { + inherits = ["_common-rocm", "_ci-rocm"] + target = "export_test_smoke" + cache-from = get_cache_from_rocm() + output = ["type=local,dest=./build/rocm-smoke-export"] +} + # Cache-only target for the source-scoped ROCm native build stage. # This persists the csrc-build stage in the registry cache even though the # final test image only consumes it indirectly while packaging the wheel. @@ -341,7 +351,13 @@ group "test-rocm-ci-with-artifacts" { # Full test image + wheel export. Kept for fallback/debugging when a pushed, # build-scoped image is useful. group "test-rocm-ci-with-wheel" { - targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"] + targets = [ + "rust-rocm-ci", + "csrc-rocm-ci", + "test-rocm-ci", + "smoke-test-rocm-ci", + "export-wheel-rocm", + ] } # Primary output tag for the ci_base build. In Buildkite this is a unique, diff --git a/tests/tools/test_docker_build_metadata_args.py b/tests/tools/test_docker_build_metadata_args.py index ed817f262e..8095ad34aa 100644 --- a/tests/tools/test_docker_build_metadata_args.py +++ b/tests/tools/test_docker_build_metadata_args.py @@ -9,6 +9,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] HELPER = REPO_ROOT / ".buildkite" / "scripts" / "docker-build-metadata-args.sh" ROCM_CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh" +ROCM_IMAGE_SMOKE = REPO_ROOT / ".buildkite" / "scripts" / "rocm" / "smoke-test-image.sh" def run_helper( @@ -198,6 +199,134 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None: assert expected in ci_bake +def test_rocm_ci_smoke_runs_in_shared_buildkit_graph() -> None: + dockerfile = (REPO_ROOT / "docker" / "Dockerfile.rocm").read_text() + ci_hcl = (REPO_ROOT / "docker" / "ci-rocm.hcl").read_text() + full_image_group = ci_hcl.split('group "test-rocm-ci-with-wheel"', maxsplit=1)[ + 1 + ].split("}", maxsplit=1)[0] + + for expected in ( + "FROM test AS test_smoke", + "smoke-test-image.sh --inside", + "FROM scratch AS export_test_smoke", + 'target "smoke-test-rocm-ci"', + 'target = "export_test_smoke"', + 'output = ["type=local,dest=./build/rocm-smoke-export"]', + ): + assert expected in dockerfile or expected in ci_hcl + assert '"smoke-test-rocm-ci"' in full_image_group + assert 'target "smoke-test-rocm-ci"' in ROCM_CI_BAKE.read_text() + + +def prepare_rocm_smoke_test( + tmp_path: Path, + *, + marker_id: str, + build_id: str, +) -> tuple[dict[str, str], Path, Path, Path]: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + docker_called = tmp_path / "docker-called" + docker = fake_bin / "docker" + docker.write_text('#!/bin/sh\ntouch "$FAKE_DOCKER_CALLED"\nexit 99\n') + docker.chmod(0o755) + + marker = tmp_path / "build" / "rocm-smoke-export" / "vllm-smoke-ok" + marker.parent.mkdir(parents=True) + marker.write_text(f"{marker_id}\n") + env = os.environ.copy() + env.update( + { + "BUILDKITE_BUILD_ID": build_id, + "FAKE_DOCKER_CALLED": str(docker_called), + "PATH": f"{fake_bin}:{env['PATH']}", + } + ) + env.pop("ROCM_CI_ARTIFACT_ONLY", None) + env.pop("VLLM_CI_SMOKE_IMAGE", None) + return env, marker, docker, docker_called + + +def test_rocm_smoke_marker_avoids_host_image_pull(tmp_path: Path) -> None: + env, marker, _, docker_called = prepare_rocm_smoke_test( + tmp_path, + marker_id="build-123", + build_id="build-123", + ) + + result = subprocess.run( + ["bash", str(ROCM_IMAGE_SMOKE)], + check=True, + cwd=tmp_path, + env=env, + stdout=subprocess.PIPE, + text=True, + ) + + assert "verified inside BuildKit" in result.stdout + assert not docker_called.exists() + assert not marker.exists() + + +def test_rocm_smoke_rejects_marker_from_another_build(tmp_path: Path) -> None: + env, marker, _, docker_called = prepare_rocm_smoke_test( + tmp_path, + marker_id="previous-build", + build_id="current-build", + ) + + result = subprocess.run( + ["bash", str(ROCM_IMAGE_SMOKE)], + check=False, + cwd=tmp_path, + env=env, + stderr=subprocess.PIPE, + text=True, + ) + + assert result.returncode == 1 + assert "previous-build, not current-build" in result.stderr + assert not docker_called.exists() + assert marker.exists() + + +def test_rocm_smoke_override_streams_current_checks_to_docker( + tmp_path: Path, +) -> None: + env, marker, docker, _ = prepare_rocm_smoke_test( + tmp_path, + marker_id="build-123", + build_id="build-123", + ) + docker_args = tmp_path / "docker-args" + docker_stdin = tmp_path / "docker-stdin" + docker.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@" > "$FAKE_DOCKER_ARGS"\n' + 'cat > "$FAKE_DOCKER_STDIN"\n' + ) + env.update( + { + "FAKE_DOCKER_ARGS": str(docker_args), + "FAKE_DOCKER_STDIN": str(docker_stdin), + "IMAGE_TAG": "rocm/vllm-ci:built", + "VLLM_CI_SMOKE_IMAGE": "rocm/vllm-ci:override", + } + ) + + subprocess.run( + ["bash", str(ROCM_IMAGE_SMOKE)], + check=True, + cwd=tmp_path, + env=env, + ) + + assert "rocm/vllm-ci:override" in docker_args.read_text().splitlines() + assert docker_args.read_text().splitlines()[-3:] == ["-s", "--", "--inside"] + assert "run_smoke_checks()" in docker_stdin.read_text() + assert marker.exists() + + def test_rocm_git_fetch_disables_automatic_maintenance(tmp_path: Path) -> None: fake_bin = tmp_path / "bin" fake_bin.mkdir()