[CI][ROCm] Avoid redundant image pulls during smoke validation (#54408)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c28feab989
commit
907b1a7f22
@@ -842,6 +842,29 @@ should_upload_wheel_artifacts() {
|
||||
|| "${TARGET}" == *"artifact"* ]]
|
||||
}
|
||||
|
||||
should_export_rocm_smoke() {
|
||||
[[ "${TARGET}" == "test-rocm-ci-with-wheel" \
|
||||
|| "${TARGET}" == "smoke-test-rocm-ci" ]]
|
||||
}
|
||||
|
||||
verify_rocm_smoke_export() {
|
||||
local marker="./build/rocm-smoke-export/vllm-smoke-ok"
|
||||
local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
|
||||
local actual_smoke_id=""
|
||||
|
||||
should_export_rocm_smoke || return 0
|
||||
if [[ ! -f "${marker}" ]]; then
|
||||
echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2
|
||||
return 1
|
||||
fi
|
||||
actual_smoke_id="$(< "${marker}")"
|
||||
if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
|
||||
echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
|
||||
>&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
get_remote_image_label() {
|
||||
local image_ref="$1"
|
||||
local label_key="$2"
|
||||
@@ -1386,8 +1409,9 @@ maybe_skip_existing_image() {
|
||||
echo "FORCE_BUILD=1 set; skipping existing-image check"
|
||||
return 0
|
||||
fi
|
||||
if ! is_ci_base_target && should_upload_wheel_artifacts; then
|
||||
echo "Artifact-producing targets always run for the current build"
|
||||
if ! is_ci_base_target \
|
||||
&& { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then
|
||||
echo "Local-output targets always run for the current build"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -1741,7 +1765,12 @@ EOF
|
||||
|
||||
uses_rocm_csrc_cache() {
|
||||
case "${TARGET}" in
|
||||
csrc-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
|
||||
csrc-rocm-ci \
|
||||
| test-rocm-ci \
|
||||
| test-rocm-ci-with-wheel \
|
||||
| test-rocm-ci-with-artifacts \
|
||||
| export-wheel-rocm \
|
||||
| smoke-test-rocm-ci)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
@@ -1752,7 +1781,12 @@ uses_rocm_csrc_cache() {
|
||||
|
||||
uses_rocm_rust_cache() {
|
||||
case "${TARGET}" in
|
||||
rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
|
||||
rust-rocm-ci \
|
||||
| test-rocm-ci \
|
||||
| test-rocm-ci-with-wheel \
|
||||
| test-rocm-ci-with-artifacts \
|
||||
| export-wheel-rocm \
|
||||
| smoke-test-rocm-ci)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
@@ -2140,7 +2174,7 @@ write_rocm_cache_override() {
|
||||
# exporter unless it is requested explicitly.
|
||||
if ((${#rust_cache_to[@]} > 0)); then
|
||||
case "${TARGET}" in
|
||||
test-rocm-ci|export-wheel-rocm)
|
||||
test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci)
|
||||
BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
|
||||
;;
|
||||
esac
|
||||
@@ -2190,6 +2224,15 @@ EOF
|
||||
cat <<EOF
|
||||
}
|
||||
|
||||
target "smoke-test-rocm-ci" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm(),
|
||||
EOF
|
||||
write_hcl_string_list " " "${combined_content_cache_from[@]}"
|
||||
cat <<EOF
|
||||
)
|
||||
}
|
||||
|
||||
target "export-wheel-rocm" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm(),
|
||||
@@ -2675,8 +2718,14 @@ main() {
|
||||
# clean prevents a failed/retried export from packaging a stale wheel.
|
||||
rm -rf ./wheel-export
|
||||
fi
|
||||
if should_export_rocm_smoke; then
|
||||
# The marker is a build output, not a cache. Never accept stale output
|
||||
# from an earlier build or retry.
|
||||
rm -rf ./build/rocm-smoke-export
|
||||
fi
|
||||
seed_dependency_caches_if_needed
|
||||
run_bake
|
||||
verify_rocm_smoke_export
|
||||
promote_stable_ci_base_tag
|
||||
publish_ci_base_handoff_ref
|
||||
upload_wheel_artifacts_if_present
|
||||
|
||||
@@ -3,6 +3,53 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
run_smoke_checks() {
|
||||
local required_dir=""
|
||||
|
||||
for required_dir in \
|
||||
/vllm-workspace \
|
||||
/vllm-workspace/tests \
|
||||
/vllm-workspace/src/vllm; do
|
||||
if [[ ! -d "${required_dir}" ]]; then
|
||||
echo "Missing directory: ${required_dir}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [[ ! -x /vllm-workspace/src/vllm/vllm-rs ]]; then
|
||||
echo "Missing executable: /vllm-workspace/src/vllm/vllm-rs" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
command -v python3
|
||||
command -v uv
|
||||
command -v pytest
|
||||
|
||||
if ! command -v amd-smi >/dev/null 2>&1 \
|
||||
&& ! command -v rocminfo >/dev/null 2>&1; then
|
||||
echo "No ROCm CLI found in image" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY'
|
||||
import torch
|
||||
import vllm
|
||||
|
||||
print(torch.__version__)
|
||||
print(vllm.__version__)
|
||||
PY
|
||||
|
||||
echo "AMD image smoke OK"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--inside" ]]; then
|
||||
run_smoke_checks
|
||||
exit
|
||||
fi
|
||||
if (($#)); then
|
||||
echo "Usage: $0 [--inside]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
|
||||
base_refreshed=""
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
@@ -14,30 +61,24 @@ if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
smoke_marker="./build/rocm-smoke-export/vllm-smoke-ok"
|
||||
expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
|
||||
if [[ -f "${smoke_marker}" \
|
||||
&& ( -z "${VLLM_CI_SMOKE_IMAGE:-}" \
|
||||
|| "${VLLM_CI_SMOKE_IMAGE}" == "${IMAGE_TAG:-}" ) ]]; then
|
||||
actual_smoke_id="$(< "${smoke_marker}")"
|
||||
if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
|
||||
echo "ROCm smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
|
||||
>&2
|
||||
exit 1
|
||||
fi
|
||||
echo "AMD image smoke OK (verified inside BuildKit)"
|
||||
rm -f -- "${smoke_marker}"
|
||||
rmdir -- "$(dirname "${smoke_marker}")" 2>/dev/null || true
|
||||
exit
|
||||
fi
|
||||
|
||||
image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
|
||||
|
||||
docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
|
||||
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
|
||||
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
|
||||
|
||||
command -v python3
|
||||
command -v uv
|
||||
command -v pytest
|
||||
|
||||
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
|
||||
echo No ROCm CLI found in image >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - <<PY
|
||||
import torch
|
||||
import vllm
|
||||
|
||||
print(torch.__version__)
|
||||
print(vllm.__version__)
|
||||
PY
|
||||
|
||||
echo AMD image smoke OK
|
||||
'
|
||||
docker run --rm -i --network=none --entrypoint /bin/bash "${image_ref}" \
|
||||
-s -- --inside < "${BASH_SOURCE[0]}"
|
||||
|
||||
@@ -899,6 +899,17 @@ RUN mkdir src && mv vllm src/vllm
|
||||
# Catch GPU<->CPU syncs in execute_model/sample_tokens during tests.
|
||||
ENV VLLM_GPU_SYNC_CHECK=error
|
||||
|
||||
# Run the structural smoke test inside BuildKit, where the test image layers
|
||||
# are already available. Export only a build-scoped success marker.
|
||||
FROM test AS test_smoke
|
||||
ARG ROCM_SMOKE_ID
|
||||
RUN --network=none \
|
||||
bash /vllm-workspace/.buildkite/scripts/rocm/smoke-test-image.sh --inside \
|
||||
&& printf '%s\n' "${ROCM_SMOKE_ID:-local}" > /vllm-smoke-ok
|
||||
|
||||
FROM scratch AS export_test_smoke
|
||||
COPY --from=test_smoke /vllm-smoke-ok /
|
||||
|
||||
# -----------------------
|
||||
# Final vLLM image
|
||||
FROM mori_base AS final_common
|
||||
|
||||
+17
-1
@@ -281,6 +281,7 @@ target "_ci-rocm" {
|
||||
args = {
|
||||
ARG_PYTORCH_ROCM_ARCH = PYTORCH_ROCM_ARCH
|
||||
CI_BASE_IMAGE = CI_BASE_IMAGE
|
||||
ROCM_SMOKE_ID = BUILDKITE_BUILD_ID
|
||||
max_jobs = CI_MAX_JOBS
|
||||
}
|
||||
}
|
||||
@@ -297,6 +298,15 @@ target "test-rocm-ci" {
|
||||
output = ["type=registry"]
|
||||
}
|
||||
|
||||
# Validate the test image in the shared BuildKit graph and export only the
|
||||
# success marker. This avoids pulling the multi-GB image into the host daemon.
|
||||
target "smoke-test-rocm-ci" {
|
||||
inherits = ["_common-rocm", "_ci-rocm"]
|
||||
target = "export_test_smoke"
|
||||
cache-from = get_cache_from_rocm()
|
||||
output = ["type=local,dest=./build/rocm-smoke-export"]
|
||||
}
|
||||
|
||||
# Cache-only target for the source-scoped ROCm native build stage.
|
||||
# This persists the csrc-build stage in the registry cache even though the
|
||||
# final test image only consumes it indirectly while packaging the wheel.
|
||||
@@ -341,7 +351,13 @@ group "test-rocm-ci-with-artifacts" {
|
||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed,
|
||||
# build-scoped image is useful.
|
||||
group "test-rocm-ci-with-wheel" {
|
||||
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||
targets = [
|
||||
"rust-rocm-ci",
|
||||
"csrc-rocm-ci",
|
||||
"test-rocm-ci",
|
||||
"smoke-test-rocm-ci",
|
||||
"export-wheel-rocm",
|
||||
]
|
||||
}
|
||||
|
||||
# Primary output tag for the ci_base build. In Buildkite this is a unique,
|
||||
|
||||
@@ -9,6 +9,7 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HELPER = REPO_ROOT / ".buildkite" / "scripts" / "docker-build-metadata-args.sh"
|
||||
ROCM_CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh"
|
||||
ROCM_IMAGE_SMOKE = REPO_ROOT / ".buildkite" / "scripts" / "rocm" / "smoke-test-image.sh"
|
||||
|
||||
|
||||
def run_helper(
|
||||
@@ -198,6 +199,134 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None:
|
||||
assert expected in ci_bake
|
||||
|
||||
|
||||
def test_rocm_ci_smoke_runs_in_shared_buildkit_graph() -> None:
|
||||
dockerfile = (REPO_ROOT / "docker" / "Dockerfile.rocm").read_text()
|
||||
ci_hcl = (REPO_ROOT / "docker" / "ci-rocm.hcl").read_text()
|
||||
full_image_group = ci_hcl.split('group "test-rocm-ci-with-wheel"', maxsplit=1)[
|
||||
1
|
||||
].split("}", maxsplit=1)[0]
|
||||
|
||||
for expected in (
|
||||
"FROM test AS test_smoke",
|
||||
"smoke-test-image.sh --inside",
|
||||
"FROM scratch AS export_test_smoke",
|
||||
'target "smoke-test-rocm-ci"',
|
||||
'target = "export_test_smoke"',
|
||||
'output = ["type=local,dest=./build/rocm-smoke-export"]',
|
||||
):
|
||||
assert expected in dockerfile or expected in ci_hcl
|
||||
assert '"smoke-test-rocm-ci"' in full_image_group
|
||||
assert 'target "smoke-test-rocm-ci"' in ROCM_CI_BAKE.read_text()
|
||||
|
||||
|
||||
def prepare_rocm_smoke_test(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
marker_id: str,
|
||||
build_id: str,
|
||||
) -> tuple[dict[str, str], Path, Path, Path]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
docker_called = tmp_path / "docker-called"
|
||||
docker = fake_bin / "docker"
|
||||
docker.write_text('#!/bin/sh\ntouch "$FAKE_DOCKER_CALLED"\nexit 99\n')
|
||||
docker.chmod(0o755)
|
||||
|
||||
marker = tmp_path / "build" / "rocm-smoke-export" / "vllm-smoke-ok"
|
||||
marker.parent.mkdir(parents=True)
|
||||
marker.write_text(f"{marker_id}\n")
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"BUILDKITE_BUILD_ID": build_id,
|
||||
"FAKE_DOCKER_CALLED": str(docker_called),
|
||||
"PATH": f"{fake_bin}:{env['PATH']}",
|
||||
}
|
||||
)
|
||||
env.pop("ROCM_CI_ARTIFACT_ONLY", None)
|
||||
env.pop("VLLM_CI_SMOKE_IMAGE", None)
|
||||
return env, marker, docker, docker_called
|
||||
|
||||
|
||||
def test_rocm_smoke_marker_avoids_host_image_pull(tmp_path: Path) -> None:
|
||||
env, marker, _, docker_called = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="build-123",
|
||||
build_id="build-123",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=True,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert "verified inside BuildKit" in result.stdout
|
||||
assert not docker_called.exists()
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_rocm_smoke_rejects_marker_from_another_build(tmp_path: Path) -> None:
|
||||
env, marker, _, docker_called = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="previous-build",
|
||||
build_id="current-build",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=False,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "previous-build, not current-build" in result.stderr
|
||||
assert not docker_called.exists()
|
||||
assert marker.exists()
|
||||
|
||||
|
||||
def test_rocm_smoke_override_streams_current_checks_to_docker(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, marker, docker, _ = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="build-123",
|
||||
build_id="build-123",
|
||||
)
|
||||
docker_args = tmp_path / "docker-args"
|
||||
docker_stdin = tmp_path / "docker-stdin"
|
||||
docker.write_text(
|
||||
'#!/bin/sh\nprintf "%s\\n" "$@" > "$FAKE_DOCKER_ARGS"\n'
|
||||
'cat > "$FAKE_DOCKER_STDIN"\n'
|
||||
)
|
||||
env.update(
|
||||
{
|
||||
"FAKE_DOCKER_ARGS": str(docker_args),
|
||||
"FAKE_DOCKER_STDIN": str(docker_stdin),
|
||||
"IMAGE_TAG": "rocm/vllm-ci:built",
|
||||
"VLLM_CI_SMOKE_IMAGE": "rocm/vllm-ci:override",
|
||||
}
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=True,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert "rocm/vllm-ci:override" in docker_args.read_text().splitlines()
|
||||
assert docker_args.read_text().splitlines()[-3:] == ["-s", "--", "--inside"]
|
||||
assert "run_smoke_checks()" in docker_stdin.read_text()
|
||||
assert marker.exists()
|
||||
|
||||
|
||||
def test_rocm_git_fetch_disables_automatic_maintenance(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
|
||||
Reference in New Issue
Block a user