[ROCm] Fix CI runtime and tests for MI355 DPX (#58244)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Mahesh Kunreddi <mahesh.kunreddi@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
mkunredd
2026-09-24 23:15:11 -05:00
committed by GitHub
co-authored by Mahesh Kunreddi Claude Andreas Karatzas Codex Andreas Karatzas
parent 44257c2e35
commit 4013f3ad6c
12 changed files with 175 additions and 47 deletions
+36 -5
View File
@@ -605,6 +605,33 @@ initialize_native_environment() {
fi
}
check_dpx_gpu_exclusivity() {
local devices=(/dev/dri/renderD*)
local device_id lock_status
if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then
echo "DPX guard requires exactly one render device; refusing to start tests." >&2
return 1
fi
# This queue mounts the same host-local HF cache into every pod on the node.
if ! mountpoint -q "${HF_HOME}"; then
echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2
return 1
fi
device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1
mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1
# Hold the descriptor through workload and teardown; never delete the file.
exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1
flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0
lock_status=$?
if [[ "${lock_status}" -eq 75 ]]; then
echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2
else
echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2
fi
return 1
}
run_native_preflight() {
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
@@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() {
# Bus data identifies a card within the public node without publishing its
# persistent UUID, serial number, or process list.
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi static -b -g all
amd-smi static --bus --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -e -k -P -x -g all
amd-smi metric --ecc --ecc-blocks --pcie --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi bad-pages -p -r -u -g all
amd-smi static --ras --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi metric -p -t -u -m -v -g all
amd-smi metric --power --temperature --usage --mem-usage --gpu all
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \
amd-smi xgmi -l -g all
amd-smi xgmi --link-status --gpu all
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \
&& command -v rocm-smi >/dev/null 2>&1; then
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
@@ -1485,6 +1512,10 @@ if is_native_runtime; then
echo "Failed to initialize the native test environment"
exit 1
fi
if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \
&& "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then
check_dpx_gpu_exclusivity || exit 1
fi
if [[ "${commands}" == *python_only_compile.sh* ]]; then
# This no-GPU job validates the ROCm precompiled/editable install path,
# rather than CPU runtime platform selection.
+6 -4
View File
@@ -364,7 +364,7 @@ def test_end_to_end(model: str):
@create_new_process_for_each_test()
def test_deep_sleep():
def test_deep_sleep(gpu_memory_cleared):
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free # in case other process is running
@@ -395,7 +395,7 @@ def test_deep_sleep():
@create_new_process_for_each_test()
def test_deep_sleep_lora():
def test_deep_sleep_lora(gpu_memory_cleared):
"""Level-2 sleep/wake/reload with enable_lora=True.
LoRA wrapping moves parameters under base_layer and adds LoRA
@@ -488,7 +488,7 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
@create_new_process_for_each_test()
def test_deep_sleep_async():
def test_deep_sleep_async(gpu_memory_cleared):
async def test():
model = "hmellor/tiny-random-LlamaForCausalLM"
free, total = torch.accelerator.get_memory_info()
@@ -527,7 +527,9 @@ def test_deep_sleep_async():
@requires_fp8
def test_deep_sleep_fp8_kvcache_mrv1(monkeypatch: pytest.MonkeyPatch):
def test_deep_sleep_fp8_kvcache_mrv1(
monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared
):
# Regression test for https://github.com/vllm-project/vllm/pull/28783.
# In particular, verify that MRV1 does not rely on post_kv_cache_wake_up()
# to restore correct output after level-2 sleep.
+18
View File
@@ -1790,6 +1790,24 @@ def disable_deepgemm_ue8m0(monkeypatch):
is_deep_gemm_e8m0_used.cache_clear()
@pytest.fixture
def gpu_memory_cleared():
"""Wait for prior tests to release GPU memory on gfx950."""
if not current_platform.is_rocm():
return
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.platforms.rocm import on_gfx950
if on_gfx950():
wait_for_gpu_memory_to_clear(
devices=[0],
threshold_ratio=0.08,
timeout_s=30,
stable_duration_s=1,
)
def _should_clean_gpu_memory_between_tests() -> bool:
# This must stay opt-in: a function-scoped fixture cannot distinguish
# stale VRAM from allocations owned by longer-lived module/session fixtures.
@@ -0,0 +1,2 @@
Qwen3-0.6B-FP8.yaml
Qwen3-1.7B-MXFP4.yaml
+31 -2
View File
@@ -12,6 +12,9 @@ from tests.kernels.quant_utils import (
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_group_quant_int8,
)
from vllm.platforms import current_platform
if current_platform.get_device_capability() < (7, 0):
@@ -64,7 +67,7 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
topk_ids = topk_ids.view(-1)
_, block_k = block_shape[0], block_shape[1]
a_q, a_s = native_per_token_group_quant_int8(a, block_k)
a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False)
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
@@ -72,7 +75,9 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype
)
act_out = SiluAndMul().forward_native(inter_out)
act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k)
act_out_q, act_out_s = native_per_token_group_quant_int8(
act_out, block_k, round_to_nearest=False
)
act_out = act_out.to(torch.float32)
out[mask] = native_w8a8_block_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype
@@ -132,3 +137,27 @@ def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
# Check results
torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
def test_block_int8_quantization_truncates_fractional_values(dtype):
"""Block INT8 quantization truncates, unlike per-token round-to-nearest."""
device = current_platform.device_type
x = torch.tensor(
[[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122],
dtype=dtype,
device=device,
)
expected = torch.tensor(
[[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device
)
out_q, scale = per_token_group_quant_int8(x, group_size=128)
ref_q, ref_s = native_per_token_group_quant_int8(
x, group_size=128, round_to_nearest=False
)
torch.testing.assert_close(out_q, expected, atol=0, rtol=0)
torch.testing.assert_close(ref_q, expected, atol=0, rtol=0)
torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0)
torch.testing.assert_close(ref_s, scale, atol=0, rtol=0)
+8 -4
View File
@@ -181,12 +181,15 @@ def native_per_token_group_quant_fp8(
return x_q, x_s
def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8):
def native_per_token_group_quant_int8(
x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch.
It converts the tensor values into int8 values and returns the
quantized tensor along with the scaling factor used for quantization.
Per-token quantization rounds to nearest; block quantization truncates.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
@@ -201,9 +204,10 @@ def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8
# Use float32 for scale calculation for stability
amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32)
x_s = amax / int8_max
x_q = (
(x_.to(torch.float32) / x_s).round().clamp(min=int8_min, max=int8_max).to(dtype)
) # Round before clamping
x_q = x_.to(torch.float32) / x_s
if round_to_nearest:
x_q = x_q.round()
x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype)
x_q = x_q.reshape(x.shape)
x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,))
@@ -614,11 +614,13 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
assert out.shape == (M, N)
assert out.dtype == torch.bfloat16
assert not torch.any(torch.isnan(out))
# Accumulate in FP32, then match the GEMM's BF16 output rounding.
A_dq = quant_dequant_mxfp4(A)
B_dq = quant_dequant_mxfp4(B)
ref = torch.matmul(A_dq.float(), B_dq.t().float())
ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float()
_print_close_stats(
f"skinny_gemm M={M} N={N} K={K}",
@@ -132,7 +132,8 @@ M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
(128, 6144, 128),
(96, 8192, 128),
(128, 12288, 128),
# Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs.
(128, 12288, 64),
(32, 12288, 128),
]
@@ -73,6 +73,7 @@ def _set_conv_state_layout(monkeypatch, layout: str) -> None:
def test_models(
hf_runner,
vllm_runner,
gpu_memory_cleared,
example_prompts,
monkeypatch,
model: str,
@@ -12,6 +12,7 @@ import gc
import pytest
import torch
from tests.utils import wait_for_gpu_memory_to_clear
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
@@ -19,6 +20,8 @@ pytestmark = pytest.mark.skipif(
reason="ROCm platform required",
)
MAX_LEAK_PCT = 10
def _mb(b: int) -> float:
return b / (1024 * 1024)
@@ -61,6 +64,18 @@ def _full_gpu_cleanup():
torch.accelerator.empty_cache()
def _wait_for_driver_memory_release(baseline: dict, peak: dict) -> None:
# Process exit can precede ROCm's release of the backing allocation.
peak_delta_mb = max(0.0, peak["drv_used_mb"] - baseline["drv_used_mb"])
target_mb = baseline["drv_used_mb"] + peak_delta_mb * MAX_LEAK_PCT / 100
wait_for_gpu_memory_to_clear(
devices=[torch.accelerator.current_device_index()],
threshold_bytes=int(target_mb * 1024**2),
timeout_s=120,
poll_interval_s=1,
)
@pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)])
def test_gpu_memory_nixl_hma(model_name, sw_size):
"""Track GPU memory through NixlConnector create/infer/shutdown cycle."""
@@ -116,7 +131,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
# shutdown + cleanup
print("\n--- shutdown ---")
llm.llm_engine.engine_core.shutdown()
llm.llm_engine.engine_core.shutdown(timeout=60)
_gpu_snapshot("3. after shutdown()", snap2["alloc_mb"])
del llm
@@ -126,6 +141,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
torch._dynamo.reset()
gc.collect()
torch.accelerator.empty_cache()
_wait_for_driver_memory_release(snap0, snap2)
snap_final = _gpu_snapshot("4. final", snap2["alloc_mb"])
# summary
@@ -160,7 +176,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size):
# Peak driver memory used above baseline
drv_peak = snap2["drv_used_mb"] - drv_base
leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0
max_leak_pct = 10
max_leak_pct = MAX_LEAK_PCT
assert leak_pct <= max_leak_pct, (
f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory "
f"not freed after NixlConnector shutdown "
@@ -196,7 +212,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
llm.generate(["hi " * 500], SamplingParams(max_tokens=1))
snap_peak = _gpu_snapshot("after generate()", snap0["alloc_mb"])
llm.llm_engine.engine_core.shutdown()
llm.llm_engine.engine_core.shutdown(timeout=60)
del llm
_full_gpu_cleanup()
cleanup_dist_env_and_memory()
@@ -204,6 +220,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
torch._dynamo.reset()
gc.collect()
torch.accelerator.empty_cache()
_wait_for_driver_memory_release(snap0, snap_peak)
snap_final = _gpu_snapshot("final", snap0["alloc_mb"])
drv_base = snap0["drv_used_mb"]
@@ -213,7 +230,7 @@ def test_gpu_memory_no_nixl_baseline(model_name):
print("=" * 90)
leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0
max_leak_pct = 10
max_leak_pct = MAX_LEAK_PCT
assert leak_pct <= max_leak_pct, (
f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory "
f"not freed after baseline shutdown "
+24 -12
View File
@@ -7,6 +7,7 @@ import pytest
from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501
DummyStatLogger,
)
from tests.utils import wait_for_memory_to_settle
from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM
from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger
@@ -30,8 +31,15 @@ async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args):
engine = AsyncLLM.from_engine_args(
log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger]
)
assert isinstance(engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger)
engine.shutdown()
try:
assert isinstance(
engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger
)
finally:
engine.shutdown()
wait_for_memory_to_settle(
threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization
)
@pytest.mark.asyncio
@@ -48,14 +56,18 @@ async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args):
disabled_log_engine_args, stat_loggers=[DummyStatLogger]
)
assert len(engine.logger_manager.stat_loggers) == 2
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
assert isinstance(
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
DummyStatLogger,
)
try:
assert len(engine.logger_manager.stat_loggers) == 2
assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1
assert isinstance(
engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0],
DummyStatLogger,
)
# log_stats is still True, since custom stat loggers are used
assert engine.log_stats
engine.shutdown()
# log_stats is still True, since custom stat loggers are used
assert engine.log_stats
finally:
engine.shutdown()
wait_for_memory_to_settle(
threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization
)
+23 -14
View File
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
import pytest
import ray
from tests.utils import wait_for_memory_to_settle
from vllm.config.model import ModelDType
from vllm.platforms import current_platform
from vllm.sampling_params import SamplingParams
@@ -46,31 +47,39 @@ def test_engine_log_metrics_ray(
) -> None:
"""Simple smoke test, verifying this can be used without exceptions.
Need to start a Ray cluster in order to verify outputs."""
engine_args = AsyncEngineArgs(
model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True
)
@ray.remote(num_gpus=1)
class EngineTestActor:
async def run(self):
engine_args = AsyncEngineArgs(
model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True
)
engine = AsyncLLM.from_engine_args(
engine_args, stat_loggers=[RayPrometheusStatLogger]
)
for i, prompt in enumerate(example_prompts):
results = engine.generate(
request_id=f"request-id-{i}",
prompt=prompt,
sampling_params=SamplingParams(max_tokens=max_tokens),
)
try:
for i, prompt in enumerate(example_prompts):
results = engine.generate(
request_id=f"request-id-{i}",
prompt=prompt,
sampling_params=SamplingParams(max_tokens=max_tokens),
)
async for _ in results:
pass
async for _ in results:
pass
finally:
engine.shutdown()
# Create the actor and call the async method
actor = EngineTestActor.remote() # type: ignore[attr-defined]
ray.get(actor.run.remote())
try:
actor = EngineTestActor.remote() # type: ignore[attr-defined]
ray.get(actor.run.remote())
finally:
ray.shutdown()
wait_for_memory_to_settle(
threshold_ratio=1.0 - engine_args.gpu_memory_utilization
)
def test_sanitized_opentelemetry_name():