[CI][ROCm] Avoid redundant image pulls during smoke validation (#54408)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c28feab989
commit
907b1a7f22
@@ -9,6 +9,7 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
HELPER = REPO_ROOT / ".buildkite" / "scripts" / "docker-build-metadata-args.sh"
|
||||
ROCM_CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh"
|
||||
ROCM_IMAGE_SMOKE = REPO_ROOT / ".buildkite" / "scripts" / "rocm" / "smoke-test-image.sh"
|
||||
|
||||
|
||||
def run_helper(
|
||||
@@ -198,6 +199,134 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None:
|
||||
assert expected in ci_bake
|
||||
|
||||
|
||||
def test_rocm_ci_smoke_runs_in_shared_buildkit_graph() -> None:
|
||||
dockerfile = (REPO_ROOT / "docker" / "Dockerfile.rocm").read_text()
|
||||
ci_hcl = (REPO_ROOT / "docker" / "ci-rocm.hcl").read_text()
|
||||
full_image_group = ci_hcl.split('group "test-rocm-ci-with-wheel"', maxsplit=1)[
|
||||
1
|
||||
].split("}", maxsplit=1)[0]
|
||||
|
||||
for expected in (
|
||||
"FROM test AS test_smoke",
|
||||
"smoke-test-image.sh --inside",
|
||||
"FROM scratch AS export_test_smoke",
|
||||
'target "smoke-test-rocm-ci"',
|
||||
'target = "export_test_smoke"',
|
||||
'output = ["type=local,dest=./build/rocm-smoke-export"]',
|
||||
):
|
||||
assert expected in dockerfile or expected in ci_hcl
|
||||
assert '"smoke-test-rocm-ci"' in full_image_group
|
||||
assert 'target "smoke-test-rocm-ci"' in ROCM_CI_BAKE.read_text()
|
||||
|
||||
|
||||
def prepare_rocm_smoke_test(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
marker_id: str,
|
||||
build_id: str,
|
||||
) -> tuple[dict[str, str], Path, Path, Path]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
docker_called = tmp_path / "docker-called"
|
||||
docker = fake_bin / "docker"
|
||||
docker.write_text('#!/bin/sh\ntouch "$FAKE_DOCKER_CALLED"\nexit 99\n')
|
||||
docker.chmod(0o755)
|
||||
|
||||
marker = tmp_path / "build" / "rocm-smoke-export" / "vllm-smoke-ok"
|
||||
marker.parent.mkdir(parents=True)
|
||||
marker.write_text(f"{marker_id}\n")
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"BUILDKITE_BUILD_ID": build_id,
|
||||
"FAKE_DOCKER_CALLED": str(docker_called),
|
||||
"PATH": f"{fake_bin}:{env['PATH']}",
|
||||
}
|
||||
)
|
||||
env.pop("ROCM_CI_ARTIFACT_ONLY", None)
|
||||
env.pop("VLLM_CI_SMOKE_IMAGE", None)
|
||||
return env, marker, docker, docker_called
|
||||
|
||||
|
||||
def test_rocm_smoke_marker_avoids_host_image_pull(tmp_path: Path) -> None:
|
||||
env, marker, _, docker_called = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="build-123",
|
||||
build_id="build-123",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=True,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert "verified inside BuildKit" in result.stdout
|
||||
assert not docker_called.exists()
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_rocm_smoke_rejects_marker_from_another_build(tmp_path: Path) -> None:
|
||||
env, marker, _, docker_called = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="previous-build",
|
||||
build_id="current-build",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=False,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "previous-build, not current-build" in result.stderr
|
||||
assert not docker_called.exists()
|
||||
assert marker.exists()
|
||||
|
||||
|
||||
def test_rocm_smoke_override_streams_current_checks_to_docker(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, marker, docker, _ = prepare_rocm_smoke_test(
|
||||
tmp_path,
|
||||
marker_id="build-123",
|
||||
build_id="build-123",
|
||||
)
|
||||
docker_args = tmp_path / "docker-args"
|
||||
docker_stdin = tmp_path / "docker-stdin"
|
||||
docker.write_text(
|
||||
'#!/bin/sh\nprintf "%s\\n" "$@" > "$FAKE_DOCKER_ARGS"\n'
|
||||
'cat > "$FAKE_DOCKER_STDIN"\n'
|
||||
)
|
||||
env.update(
|
||||
{
|
||||
"FAKE_DOCKER_ARGS": str(docker_args),
|
||||
"FAKE_DOCKER_STDIN": str(docker_stdin),
|
||||
"IMAGE_TAG": "rocm/vllm-ci:built",
|
||||
"VLLM_CI_SMOKE_IMAGE": "rocm/vllm-ci:override",
|
||||
}
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
["bash", str(ROCM_IMAGE_SMOKE)],
|
||||
check=True,
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert "rocm/vllm-ci:override" in docker_args.read_text().splitlines()
|
||||
assert docker_args.read_text().splitlines()[-3:] == ["-s", "--", "--inside"]
|
||||
assert "run_smoke_checks()" in docker_stdin.read_text()
|
||||
assert marker.exists()
|
||||
|
||||
|
||||
def test_rocm_git_fetch_disables_automatic_maintenance(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
|
||||
Reference in New Issue
Block a user