[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-11 14:19:16 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 4f2f31b82e
commit 1ab2801dde
56 changed files with 500 additions and 558 deletions
+1 -4
View File
@@ -12,7 +12,7 @@ import pytest
import pytest_asyncio
import requests
from tests.utils import ROCM_ENV_OVERRIDES, RemoteOpenAIServer
from tests.utils import RemoteOpenAIServer
from tests.v1.utils import check_request_balancing
from vllm.platforms import current_platform
@@ -186,7 +186,6 @@ class MultinodeInternalLBServerManager:
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
**ROCM_ENV_OVERRIDES,
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(r, r + gpus_per_node)
@@ -308,7 +307,6 @@ class APIOnlyServerManager:
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
**ROCM_ENV_OVERRIDES,
# No GPUs needed for API-only server
},
)
@@ -329,7 +327,6 @@ class APIOnlyServerManager:
engines_server_args,
auto_port=False,
env_dict={
**ROCM_ENV_OVERRIDES,
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(self.dp_size * self.tp_size)
@@ -38,6 +38,19 @@ SIMPLE_PROMPT = (
MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")
def _meets_accuracy_threshold(measured_value: float, expected_value: float) -> bool:
return measured_value >= expected_value - RTOL
def test_accuracy_threshold_is_one_sided():
expected_value = 0.77
minimum_value = expected_value - RTOL
assert _meets_accuracy_threshold(minimum_value, expected_value)
assert _meets_accuracy_threshold(expected_value + RTOL + 0.01, expected_value)
assert not _meets_accuracy_threshold(minimum_value - 0.01, expected_value)
def run_simple_prompt():
client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL)
completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT)
@@ -93,7 +106,7 @@ def test_accuracy():
)
return
assert (
measured_value - RTOL < expected_value
and measured_value + RTOL > expected_value
), f"Expected: {expected_value} | Measured: {measured_value}"
minimum_value = expected_value - RTOL
assert _meets_accuracy_threshold(measured_value, expected_value), (
f"Expected at least: {minimum_value} | Measured: {measured_value}"
)
@@ -18,6 +18,7 @@ from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
CanonicalKVCacheTensor,
GPULoadStoreSpec,
TransferResult,
)
from vllm.v1.kv_offload.cpu import gpu_worker
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
@@ -612,3 +613,72 @@ def test_transfer_multi_group(
)
worker.shutdown()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="stream ordering test requires a CUDA-like platform",
)
@torch.inference_mode()
def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> None:
"""A CPU load must land after pending writes to its GPU destination."""
device = DEVICES[0]
page_size_bytes = 128 * 1024
num_blocks = 64
loaded_blocks = list(range(32))
sentinel = 0x5A
gpu_tensor = torch.zeros(
(num_blocks, page_size_bytes), dtype=torch.int8, device=device
)
loaded_block_ids = torch.tensor(loaded_blocks, dtype=torch.long, device=device)
worker = CPUOffloadingWorker(
kv_caches=CanonicalKVCaches(
tensors=[
CanonicalKVCacheTensor(
tensor=gpu_tensor, page_size_bytes=page_size_bytes
)
],
group_data_refs=[
[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=page_size_bytes)]
],
),
blocks_per_chunk=1,
num_cpu_blocks=num_blocks,
)
worker._load_handler.src_tensors[0].fill_(sentinel)
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
try:
for trial in range(3):
gpu_tensor.fill_(0x11)
torch.accelerator.synchronize()
# Model a delayed zero of a freshly allocated KV block. Without a
# compute-stream dependency, the DMA can finish during the sleep
# and this later fill wipes out the loaded cache contents.
torch.cuda._sleep(50_000_000)
gpu_tensor.index_fill_(0, loaded_block_ids, 0)
assert worker.submit_load(
trial + 1,
CPULoadStoreSpec(loaded_blocks),
GPULoadStoreSpec(
loaded_blocks,
group_sizes=(len(loaded_blocks),),
block_indices=(0,),
),
)
deadline = time.time() + 10
finished: list[TransferResult] = []
while time.time() < deadline and not finished:
finished = worker.get_finished()
if not finished:
time.sleep(0.001)
assert finished and finished[0].success, f"load {trial} did not finish"
torch.accelerator.synchronize()
for block_id in loaded_blocks:
torch.testing.assert_close(gpu_tensor[block_id].cpu(), expected)
finally:
worker.shutdown()
-12
View File
@@ -1114,7 +1114,6 @@ def test_correct_decoded_token_preserves_valid_tokens():
def test_spec_decode_logprobs(
logprobs_mode: LogprobsMode,
model_setup: tuple[str, str, dict, int],
monkeypatch,
):
"""Spec decode logprobs should match those of the base model.
@@ -1127,20 +1126,9 @@ def test_spec_decode_logprobs(
logprobs_mode: logprobs mode.
model_setup: Tuple of (method, base model name,
speculative_config dict, top_logprobs).
monkeypatch: pytest fixture for setting env vars.
"""
from vllm import LLM
# The ROCm skinny GEMM kernels (gemm_kernels.cu) are
# non-deterministic across LLM instantiations due to persistent
# workgroup scheduling and wave-level shuffle reductions, which
# causes logprob differences that get misattributed to spec decode.
# Disable them so this test isolates spec decode correctness only.
# TODO(akaratza): Remove this workaround once the follow-up to
# https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
# lands with a determinism fix for wvSplitK kernels.
monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")
method, model_name, spec_config, top_logprobs = model_setup
prompt = "Hello world " * 50
+60
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from types import SimpleNamespace
from unittest.mock import Mock
@@ -126,6 +127,34 @@ def get_vllm_config():
return vllm_config
@pytest.mark.parametrize("gc_initially_enabled", [True, False])
def test_freeze_gc_disables_and_restores_automatic_gc(
monkeypatch: pytest.MonkeyPatch,
gc_initially_enabled: bool,
):
monkeypatch.setenv("VLLM_ENABLE_CUDAGRAPH_GC", "0")
original_gc_state = gc.isenabled()
try:
if gc_initially_enabled:
gc.enable()
else:
gc.disable()
with (
pytest.raises(RuntimeError, match="capture failed"),
GPUModelRunner._freeze_gc(),
):
assert not gc.isenabled()
raise RuntimeError("capture failed")
assert gc.isenabled() is gc_initially_enabled
finally:
if original_gc_state:
gc.enable()
else:
gc.disable()
@pytest.fixture
def model_runner():
vllm_config = get_vllm_config()
@@ -896,6 +925,37 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
assert torch.equal(passed_draft_probs, expected_draft_probs)
def test_dummy_sampler_run_warms_all_greedy_rejection_sampler(monkeypatch):
runner = object.__new__(GPUModelRunner)
runner.device = torch.device("cpu")
runner.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(multimodal_config=None)
)
runner.model = SimpleNamespace(compute_logits=Mock(return_value=torch.randn(3, 8)))
runner.sampler = Mock(return_value="sampler_output")
runner.sampler.logprobs_mode = "processed_logprobs"
runner.speculative_config = SimpleNamespace(
rejection_sample_method="standard",
draft_sample_method="greedy",
)
runner.rejection_sampler = Mock()
synchronize = Mock()
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
output = GPUModelRunner._dummy_sampler_run(runner, torch.randn(3, 4))
assert output == "sampler_output"
assert runner.rejection_sampler.call_count == 2
mixed_metadata = runner.rejection_sampler.call_args_list[0].args[3]
all_greedy_metadata = runner.rejection_sampler.call_args_list[1].args[3]
assert not mixed_metadata.all_greedy
assert mixed_metadata.temperature is not None
assert all_greedy_metadata.all_greedy
assert not all_greedy_metadata.all_random
assert all_greedy_metadata.temperature is None
synchronize.assert_called_once_with()
def test_invalid_draft_suffixes_remain_rejected_in_metadata():
runner = object.__new__(GPUModelRunner)
runner.device = torch.device("cpu")