diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 2705c3b87e..d07f3b3830 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -605,6 +605,33 @@ initialize_native_environment() { fi } +check_dpx_gpu_exclusivity() { + local devices=(/dev/dri/renderD*) + local device_id lock_status + + if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then + echo "DPX guard requires exactly one render device; refusing to start tests." >&2 + return 1 + fi + # This queue mounts the same host-local HF cache into every pod on the node. + if ! mountpoint -q "${HF_HOME}"; then + echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2 + return 1 + fi + device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1 + mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1 + # Hold the descriptor through workload and teardown; never delete the file. + exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1 + flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0 + lock_status=$? + if [[ "${lock_status}" -eq 75 ]]; then + echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2 + else + echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2 + fi + return 1 +} + run_native_preflight() { local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" @@ -1166,15 +1193,15 @@ collect_rocm_failure_diagnostics() { # Bus data identifies a card within the public node without publishing its # persistent UUID, serial number, or process list. run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi static -b -g all + amd-smi static --bus --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi metric -e -k -P -x -g all + amd-smi metric --ecc --ecc-blocks --pcie --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi bad-pages -p -r -u -g all + amd-smi static --ras --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi metric -p -t -u -m -v -g all + amd-smi metric --power --temperature --usage --mem-usage --gpu all run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ - amd-smi xgmi -l -g all + amd-smi xgmi --link-status --gpu all elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ && command -v rocm-smi >/dev/null 2>&1; then run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ @@ -1485,6 +1512,10 @@ if is_native_runtime; then echo "Failed to initialize the native test environment" exit 1 fi + if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \ + && "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then + check_dpx_gpu_exclusivity || exit 1 + fi if [[ "${commands}" == *python_only_compile.sh* ]]; then # This no-GPU job validates the ROCm precompiled/editable install path, # rather than CPU runtime platform selection. diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index 6a483d26aa..3df281e8e2 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -364,7 +364,7 @@ def test_end_to_end(model: str): @create_new_process_for_each_test() -def test_deep_sleep(): +def test_deep_sleep(gpu_memory_cleared): model = "hmellor/tiny-random-LlamaForCausalLM" free, total = torch.accelerator.get_memory_info() used_bytes_baseline = total - free # in case other process is running @@ -395,7 +395,7 @@ def test_deep_sleep(): @create_new_process_for_each_test() -def test_deep_sleep_lora(): +def test_deep_sleep_lora(gpu_memory_cleared): """Level-2 sleep/wake/reload with enable_lora=True. LoRA wrapping moves parameters under base_layer and adds LoRA @@ -488,7 +488,7 @@ def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch): @create_new_process_for_each_test() -def test_deep_sleep_async(): +def test_deep_sleep_async(gpu_memory_cleared): async def test(): model = "hmellor/tiny-random-LlamaForCausalLM" free, total = torch.accelerator.get_memory_info() @@ -527,7 +527,9 @@ def test_deep_sleep_async(): @requires_fp8 -def test_deep_sleep_fp8_kvcache_mrv1(monkeypatch: pytest.MonkeyPatch): +def test_deep_sleep_fp8_kvcache_mrv1( + monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared +): # Regression test for https://github.com/vllm-project/vllm/pull/28783. # In particular, verify that MRV1 does not rely on post_kv_cache_wake_up() # to restore correct output after level-2 sleep. diff --git a/tests/conftest.py b/tests/conftest.py index 45c629b875..de5038b981 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1790,6 +1790,24 @@ def disable_deepgemm_ue8m0(monkeypatch): is_deep_gemm_e8m0_used.cache_clear() +@pytest.fixture +def gpu_memory_cleared(): + """Wait for prior tests to release GPU memory on gfx950.""" + if not current_platform.is_rocm(): + return + + from tests.utils import wait_for_gpu_memory_to_clear + from vllm.platforms.rocm import on_gfx950 + + if on_gfx950(): + wait_for_gpu_memory_to_clear( + devices=[0], + threshold_ratio=0.08, + timeout_s=30, + stable_duration_s=1, + ) + + def _should_clean_gpu_memory_between_tests() -> bool: # This must stay opt-in: a function-scoped fixture cannot distinguish # stale VRAM from allocations owned by longer-lived module/session fixtures. diff --git a/tests/evals/gsm8k/configs/models-gfx950-small.txt b/tests/evals/gsm8k/configs/models-gfx950-small.txt new file mode 100644 index 0000000000..d818a067af --- /dev/null +++ b/tests/evals/gsm8k/configs/models-gfx950-small.txt @@ -0,0 +1,2 @@ +Qwen3-0.6B-FP8.yaml +Qwen3-1.7B-MXFP4.yaml diff --git a/tests/kernels/moe/test_block_int8.py b/tests/kernels/moe/test_block_int8.py index 415faafb3a..5e853834bb 100644 --- a/tests/kernels/moe/test_block_int8.py +++ b/tests/kernels/moe/test_block_int8.py @@ -12,6 +12,9 @@ from tests.kernels.quant_utils import ( from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk +from vllm.model_executor.layers.quantization.utils.int8_utils import ( + per_token_group_quant_int8, +) from vllm.platforms import current_platform if current_platform.get_device_capability() < (7, 0): @@ -64,7 +67,7 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): topk_ids = topk_ids.view(-1) _, block_k = block_shape[0], block_shape[1] - a_q, a_s = native_per_token_group_quant_int8(a, block_k) + a_q, a_s = native_per_token_group_quant_int8(a, block_k, round_to_nearest=False) for i in range(w1.shape[0]): mask = topk_ids == i if mask.sum(): @@ -72,7 +75,9 @@ def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype ) act_out = SiluAndMul().forward_native(inter_out) - act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k) + act_out_q, act_out_s = native_per_token_group_quant_int8( + act_out, block_k, round_to_nearest=False + ) act_out = act_out.to(torch.float32) out[mask] = native_w8a8_block_matmul( act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype @@ -132,3 +137,27 @@ def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed): # Check results torch.testing.assert_close(out, ref_out, atol=0.065, rtol=0.065) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +def test_block_int8_quantization_truncates_fractional_values(dtype): + """Block INT8 quantization truncates, unlike per-token round-to-nearest.""" + device = current_platform.device_type + x = torch.tensor( + [[-127, -1.75, -0.75, 0.75, 1.75, 127] + [0] * 122], + dtype=dtype, + device=device, + ) + expected = torch.tensor( + [[-127, -1, 0, 0, 1, 127] + [0] * 122], dtype=torch.int8, device=device + ) + + out_q, scale = per_token_group_quant_int8(x, group_size=128) + ref_q, ref_s = native_per_token_group_quant_int8( + x, group_size=128, round_to_nearest=False + ) + + torch.testing.assert_close(out_q, expected, atol=0, rtol=0) + torch.testing.assert_close(ref_q, expected, atol=0, rtol=0) + torch.testing.assert_close(scale, torch.ones_like(scale), atol=0, rtol=0) + torch.testing.assert_close(ref_s, scale, atol=0, rtol=0) diff --git a/tests/kernels/quant_utils.py b/tests/kernels/quant_utils.py index 7e14ec95d3..46981e565d 100644 --- a/tests/kernels/quant_utils.py +++ b/tests/kernels/quant_utils.py @@ -181,12 +181,15 @@ def native_per_token_group_quant_fp8( return x_q, x_s -def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8): +def native_per_token_group_quant_int8( + x, group_size, eps=1e-10, dtype=torch.int8, *, round_to_nearest=True +): """Function to perform per-token-group quantization on an input tensor `x` using native torch. It converts the tensor values into int8 values and returns the quantized tensor along with the scaling factor used for quantization. + Per-token quantization rounds to nearest; block quantization truncates. """ assert x.shape[-1] % group_size == 0, ( "the last dimension of `x` must be divisible by `group_size`" @@ -201,9 +204,10 @@ def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8 # Use float32 for scale calculation for stability amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) x_s = amax / int8_max - x_q = ( - (x_.to(torch.float32) / x_s).round().clamp(min=int8_min, max=int8_max).to(dtype) - ) # Round before clamping + x_q = x_.to(torch.float32) / x_s + if round_to_nearest: + x_q = x_q.round() + x_q = x_q.clamp(min=int8_min, max=int8_max).to(dtype) x_q = x_q.reshape(x.shape) x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index d4e16629af..b2547c080a 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -614,11 +614,13 @@ def test_aiter_fp4_gemm_skinny_shapes(M, N, K): out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale) assert out.shape == (M, N) + assert out.dtype == torch.bfloat16 assert not torch.any(torch.isnan(out)) + # Accumulate in FP32, then match the GEMM's BF16 output rounding. A_dq = quant_dequant_mxfp4(A) B_dq = quant_dequant_mxfp4(B) - ref = torch.matmul(A_dq.float(), B_dq.t().float()) + ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float() _print_close_stats( f"skinny_gemm M={M} N={N} K={K}", diff --git a/tests/kernels/quantization/test_rocm_skinny_gemms.py b/tests/kernels/quantization/test_rocm_skinny_gemms.py index 7a7fe326bf..838ce34419 100644 --- a/tests/kernels/quantization/test_rocm_skinny_gemms.py +++ b/tests/kernels/quantization/test_rocm_skinny_gemms.py @@ -132,7 +132,8 @@ M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16] NKM_FACTORS_WVSPLITKRC_LARGE_K = [ (128, 6144, 128), (96, 8192, 128), - (128, 12288, 128), + # Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs. + (128, 12288, 64), (32, 12288, 128), ] diff --git a/tests/models/language/generation/test_hybrid.py b/tests/models/language/generation/test_hybrid.py index b77670956a..1b90429feb 100644 --- a/tests/models/language/generation/test_hybrid.py +++ b/tests/models/language/generation/test_hybrid.py @@ -73,6 +73,7 @@ def _set_conv_state_layout(monkeypatch, layout: str) -> None: def test_models( hf_runner, vllm_runner, + gpu_memory_cleared, example_prompts, monkeypatch, model: str, diff --git a/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py b/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py index da8be07157..5d617a7ec3 100644 --- a/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py +++ b/tests/v1/kv_connector/unit/test_nixl_rocm_gpu_mem_diag.py @@ -12,6 +12,7 @@ import gc import pytest import torch +from tests.utils import wait_for_gpu_memory_to_clear from vllm.platforms import current_platform pytestmark = pytest.mark.skipif( @@ -19,6 +20,8 @@ pytestmark = pytest.mark.skipif( reason="ROCm platform required", ) +MAX_LEAK_PCT = 10 + def _mb(b: int) -> float: return b / (1024 * 1024) @@ -61,6 +64,18 @@ def _full_gpu_cleanup(): torch.accelerator.empty_cache() +def _wait_for_driver_memory_release(baseline: dict, peak: dict) -> None: + # Process exit can precede ROCm's release of the backing allocation. + peak_delta_mb = max(0.0, peak["drv_used_mb"] - baseline["drv_used_mb"]) + target_mb = baseline["drv_used_mb"] + peak_delta_mb * MAX_LEAK_PCT / 100 + wait_for_gpu_memory_to_clear( + devices=[torch.accelerator.current_device_index()], + threshold_bytes=int(target_mb * 1024**2), + timeout_s=120, + poll_interval_s=1, + ) + + @pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)]) def test_gpu_memory_nixl_hma(model_name, sw_size): """Track GPU memory through NixlConnector create/infer/shutdown cycle.""" @@ -116,7 +131,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): # shutdown + cleanup print("\n--- shutdown ---") - llm.llm_engine.engine_core.shutdown() + llm.llm_engine.engine_core.shutdown(timeout=60) _gpu_snapshot("3. after shutdown()", snap2["alloc_mb"]) del llm @@ -126,6 +141,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): torch._dynamo.reset() gc.collect() torch.accelerator.empty_cache() + _wait_for_driver_memory_release(snap0, snap2) snap_final = _gpu_snapshot("4. final", snap2["alloc_mb"]) # summary @@ -160,7 +176,7 @@ def test_gpu_memory_nixl_hma(model_name, sw_size): # Peak driver memory used above baseline drv_peak = snap2["drv_used_mb"] - drv_base leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0 - max_leak_pct = 10 + max_leak_pct = MAX_LEAK_PCT assert leak_pct <= max_leak_pct, ( f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory " f"not freed after NixlConnector shutdown " @@ -196,7 +212,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): llm.generate(["hi " * 500], SamplingParams(max_tokens=1)) snap_peak = _gpu_snapshot("after generate()", snap0["alloc_mb"]) - llm.llm_engine.engine_core.shutdown() + llm.llm_engine.engine_core.shutdown(timeout=60) del llm _full_gpu_cleanup() cleanup_dist_env_and_memory() @@ -204,6 +220,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): torch._dynamo.reset() gc.collect() torch.accelerator.empty_cache() + _wait_for_driver_memory_release(snap0, snap_peak) snap_final = _gpu_snapshot("final", snap0["alloc_mb"]) drv_base = snap0["drv_used_mb"] @@ -213,7 +230,7 @@ def test_gpu_memory_no_nixl_baseline(model_name): print("=" * 90) leak_pct = (drv_leaked / drv_peak * 100) if drv_peak > 0 else 0 - max_leak_pct = 10 + max_leak_pct = MAX_LEAK_PCT assert leak_pct <= max_leak_pct, ( f"{drv_leaked:.0f} MB ({leak_pct:.1f}%) of driver-level GPU memory " f"not freed after baseline shutdown " diff --git a/tests/v1/metrics/test_engine_logger_apis.py b/tests/v1/metrics/test_engine_logger_apis.py index 5741df0d56..e8fc1375e2 100644 --- a/tests/v1/metrics/test_engine_logger_apis.py +++ b/tests/v1/metrics/test_engine_logger_apis.py @@ -7,6 +7,7 @@ import pytest from tests.plugins.vllm_add_dummy_stat_logger.dummy_stat_logger.dummy_stat_logger import ( # noqa E501 DummyStatLogger, ) +from tests.utils import wait_for_memory_to_settle from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM from vllm.v1.metrics.ray_wrappers import RayPrometheusStatLogger @@ -30,8 +31,15 @@ async def test_async_llm_replace_default_loggers(log_stats_enabled_engine_args): engine = AsyncLLM.from_engine_args( log_stats_enabled_engine_args, stat_loggers=[RayPrometheusStatLogger] ) - assert isinstance(engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger) - engine.shutdown() + try: + assert isinstance( + engine.logger_manager.stat_loggers[0], RayPrometheusStatLogger + ) + finally: + engine.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - log_stats_enabled_engine_args.gpu_memory_utilization + ) @pytest.mark.asyncio @@ -48,14 +56,18 @@ async def test_async_llm_add_to_default_loggers(log_stats_enabled_engine_args): disabled_log_engine_args, stat_loggers=[DummyStatLogger] ) - assert len(engine.logger_manager.stat_loggers) == 2 - assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 - assert isinstance( - engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], - DummyStatLogger, - ) + try: + assert len(engine.logger_manager.stat_loggers) == 2 + assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 + assert isinstance( + engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], + DummyStatLogger, + ) - # log_stats is still True, since custom stat loggers are used - assert engine.log_stats - - engine.shutdown() + # log_stats is still True, since custom stat loggers are used + assert engine.log_stats + finally: + engine.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - disabled_log_engine_args.gpu_memory_utilization + ) diff --git a/tests/v1/metrics/test_ray_metrics.py b/tests/v1/metrics/test_ray_metrics.py index ce4ebcbfdb..8552e896c0 100644 --- a/tests/v1/metrics/test_ray_metrics.py +++ b/tests/v1/metrics/test_ray_metrics.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pytest import ray +from tests.utils import wait_for_memory_to_settle from vllm.config.model import ModelDType from vllm.platforms import current_platform from vllm.sampling_params import SamplingParams @@ -46,31 +47,39 @@ def test_engine_log_metrics_ray( ) -> None: """Simple smoke test, verifying this can be used without exceptions. Need to start a Ray cluster in order to verify outputs.""" + engine_args = AsyncEngineArgs( + model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True + ) @ray.remote(num_gpus=1) class EngineTestActor: async def run(self): - engine_args = AsyncEngineArgs( - model=model, dtype=dtype, disable_log_stats=False, enforce_eager=True - ) - engine = AsyncLLM.from_engine_args( engine_args, stat_loggers=[RayPrometheusStatLogger] ) - for i, prompt in enumerate(example_prompts): - results = engine.generate( - request_id=f"request-id-{i}", - prompt=prompt, - sampling_params=SamplingParams(max_tokens=max_tokens), - ) + try: + for i, prompt in enumerate(example_prompts): + results = engine.generate( + request_id=f"request-id-{i}", + prompt=prompt, + sampling_params=SamplingParams(max_tokens=max_tokens), + ) - async for _ in results: - pass + async for _ in results: + pass + finally: + engine.shutdown() # Create the actor and call the async method - actor = EngineTestActor.remote() # type: ignore[attr-defined] - ray.get(actor.run.remote()) + try: + actor = EngineTestActor.remote() # type: ignore[attr-defined] + ray.get(actor.run.remote()) + finally: + ray.shutdown() + wait_for_memory_to_settle( + threshold_ratio=1.0 - engine_args.gpu_memory_utilization + ) def test_sanitized_opentelemetry_name():