[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f2f31b82e
commit
1ab2801dde
@@ -12,7 +12,7 @@ import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, RemoteOpenAIServer
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from tests.v1.utils import check_request_balancing
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -186,7 +186,6 @@ class MultinodeInternalLBServerManager:
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
**ROCM_ENV_OVERRIDES,
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(r, r + gpus_per_node)
|
||||
@@ -308,7 +307,6 @@ class APIOnlyServerManager:
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
**ROCM_ENV_OVERRIDES,
|
||||
# No GPUs needed for API-only server
|
||||
},
|
||||
)
|
||||
@@ -329,7 +327,6 @@ class APIOnlyServerManager:
|
||||
engines_server_args,
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
**ROCM_ENV_OVERRIDES,
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(self.dp_size * self.tp_size)
|
||||
|
||||
@@ -38,6 +38,19 @@ SIMPLE_PROMPT = (
|
||||
MODEL_NAME = os.environ.get("TEST_MODEL", "Qwen/Qwen3-0.6B")
|
||||
|
||||
|
||||
def _meets_accuracy_threshold(measured_value: float, expected_value: float) -> bool:
|
||||
return measured_value >= expected_value - RTOL
|
||||
|
||||
|
||||
def test_accuracy_threshold_is_one_sided():
|
||||
expected_value = 0.77
|
||||
minimum_value = expected_value - RTOL
|
||||
|
||||
assert _meets_accuracy_threshold(minimum_value, expected_value)
|
||||
assert _meets_accuracy_threshold(expected_value + RTOL + 0.01, expected_value)
|
||||
assert not _meets_accuracy_threshold(minimum_value - 0.01, expected_value)
|
||||
|
||||
|
||||
def run_simple_prompt():
|
||||
client = openai.OpenAI(api_key="EMPTY", base_url=BASE_URL)
|
||||
completion = client.completions.create(model=MODEL_NAME, prompt=SIMPLE_PROMPT)
|
||||
@@ -93,7 +106,7 @@ def test_accuracy():
|
||||
)
|
||||
return
|
||||
|
||||
assert (
|
||||
measured_value - RTOL < expected_value
|
||||
and measured_value + RTOL > expected_value
|
||||
), f"Expected: {expected_value} | Measured: {measured_value}"
|
||||
minimum_value = expected_value - RTOL
|
||||
assert _meets_accuracy_threshold(measured_value, expected_value), (
|
||||
f"Expected at least: {minimum_value} | Measured: {measured_value}"
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
CanonicalKVCacheTensor,
|
||||
GPULoadStoreSpec,
|
||||
TransferResult,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu import gpu_worker
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
@@ -612,3 +613,72 @@ def test_transfer_multi_group(
|
||||
)
|
||||
|
||||
worker.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="stream ordering test requires a CUDA-like platform",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_load_waits_for_pending_compute_stream_writes(default_vllm_config) -> None:
|
||||
"""A CPU load must land after pending writes to its GPU destination."""
|
||||
device = DEVICES[0]
|
||||
page_size_bytes = 128 * 1024
|
||||
num_blocks = 64
|
||||
loaded_blocks = list(range(32))
|
||||
sentinel = 0x5A
|
||||
|
||||
gpu_tensor = torch.zeros(
|
||||
(num_blocks, page_size_bytes), dtype=torch.int8, device=device
|
||||
)
|
||||
loaded_block_ids = torch.tensor(loaded_blocks, dtype=torch.long, device=device)
|
||||
worker = CPUOffloadingWorker(
|
||||
kv_caches=CanonicalKVCaches(
|
||||
tensors=[
|
||||
CanonicalKVCacheTensor(
|
||||
tensor=gpu_tensor, page_size_bytes=page_size_bytes
|
||||
)
|
||||
],
|
||||
group_data_refs=[
|
||||
[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=page_size_bytes)]
|
||||
],
|
||||
),
|
||||
blocks_per_chunk=1,
|
||||
num_cpu_blocks=num_blocks,
|
||||
)
|
||||
worker._load_handler.src_tensors[0].fill_(sentinel)
|
||||
expected = torch.full((page_size_bytes,), sentinel, dtype=torch.int8)
|
||||
|
||||
try:
|
||||
for trial in range(3):
|
||||
gpu_tensor.fill_(0x11)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Model a delayed zero of a freshly allocated KV block. Without a
|
||||
# compute-stream dependency, the DMA can finish during the sleep
|
||||
# and this later fill wipes out the loaded cache contents.
|
||||
torch.cuda._sleep(50_000_000)
|
||||
gpu_tensor.index_fill_(0, loaded_block_ids, 0)
|
||||
|
||||
assert worker.submit_load(
|
||||
trial + 1,
|
||||
CPULoadStoreSpec(loaded_blocks),
|
||||
GPULoadStoreSpec(
|
||||
loaded_blocks,
|
||||
group_sizes=(len(loaded_blocks),),
|
||||
block_indices=(0,),
|
||||
),
|
||||
)
|
||||
deadline = time.time() + 10
|
||||
finished: list[TransferResult] = []
|
||||
while time.time() < deadline and not finished:
|
||||
finished = worker.get_finished()
|
||||
if not finished:
|
||||
time.sleep(0.001)
|
||||
assert finished and finished[0].success, f"load {trial} did not finish"
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
for block_id in loaded_blocks:
|
||||
torch.testing.assert_close(gpu_tensor[block_id].cpu(), expected)
|
||||
finally:
|
||||
worker.shutdown()
|
||||
|
||||
@@ -1114,7 +1114,6 @@ def test_correct_decoded_token_preserves_valid_tokens():
|
||||
def test_spec_decode_logprobs(
|
||||
logprobs_mode: LogprobsMode,
|
||||
model_setup: tuple[str, str, dict, int],
|
||||
monkeypatch,
|
||||
):
|
||||
"""Spec decode logprobs should match those of the base model.
|
||||
|
||||
@@ -1127,20 +1126,9 @@ def test_spec_decode_logprobs(
|
||||
logprobs_mode: logprobs mode.
|
||||
model_setup: Tuple of (method, base model name,
|
||||
speculative_config dict, top_logprobs).
|
||||
monkeypatch: pytest fixture for setting env vars.
|
||||
"""
|
||||
from vllm import LLM
|
||||
|
||||
# The ROCm skinny GEMM kernels (gemm_kernels.cu) are
|
||||
# non-deterministic across LLM instantiations due to persistent
|
||||
# workgroup scheduling and wave-level shuffle reductions, which
|
||||
# causes logprob differences that get misattributed to spec decode.
|
||||
# Disable them so this test isolates spec decode correctness only.
|
||||
# TODO(akaratza): Remove this workaround once the follow-up to
|
||||
# https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
|
||||
# lands with a determinism fix for wvSplitK kernels.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0")
|
||||
|
||||
method, model_name, spec_config, top_logprobs = model_setup
|
||||
|
||||
prompt = "Hello world " * 50
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -126,6 +127,34 @@ def get_vllm_config():
|
||||
return vllm_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gc_initially_enabled", [True, False])
|
||||
def test_freeze_gc_disables_and_restores_automatic_gc(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
gc_initially_enabled: bool,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_CUDAGRAPH_GC", "0")
|
||||
original_gc_state = gc.isenabled()
|
||||
try:
|
||||
if gc_initially_enabled:
|
||||
gc.enable()
|
||||
else:
|
||||
gc.disable()
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="capture failed"),
|
||||
GPUModelRunner._freeze_gc(),
|
||||
):
|
||||
assert not gc.isenabled()
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
assert gc.isenabled() is gc_initially_enabled
|
||||
finally:
|
||||
if original_gc_state:
|
||||
gc.enable()
|
||||
else:
|
||||
gc.disable()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_runner():
|
||||
vllm_config = get_vllm_config()
|
||||
@@ -896,6 +925,37 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
|
||||
assert torch.equal(passed_draft_probs, expected_draft_probs)
|
||||
|
||||
|
||||
def test_dummy_sampler_run_warms_all_greedy_rejection_sampler(monkeypatch):
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(multimodal_config=None)
|
||||
)
|
||||
runner.model = SimpleNamespace(compute_logits=Mock(return_value=torch.randn(3, 8)))
|
||||
runner.sampler = Mock(return_value="sampler_output")
|
||||
runner.sampler.logprobs_mode = "processed_logprobs"
|
||||
runner.speculative_config = SimpleNamespace(
|
||||
rejection_sample_method="standard",
|
||||
draft_sample_method="greedy",
|
||||
)
|
||||
runner.rejection_sampler = Mock()
|
||||
synchronize = Mock()
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
|
||||
|
||||
output = GPUModelRunner._dummy_sampler_run(runner, torch.randn(3, 4))
|
||||
|
||||
assert output == "sampler_output"
|
||||
assert runner.rejection_sampler.call_count == 2
|
||||
mixed_metadata = runner.rejection_sampler.call_args_list[0].args[3]
|
||||
all_greedy_metadata = runner.rejection_sampler.call_args_list[1].args[3]
|
||||
assert not mixed_metadata.all_greedy
|
||||
assert mixed_metadata.temperature is not None
|
||||
assert all_greedy_metadata.all_greedy
|
||||
assert not all_greedy_metadata.all_random
|
||||
assert all_greedy_metadata.temperature is None
|
||||
synchronize.assert_called_once_with()
|
||||
|
||||
|
||||
def test_invalid_draft_suffixes_remain_rejected_in_metadata():
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
|
||||
Reference in New Issue
Block a user