[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-11 14:19:16 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 4f2f31b82e
commit 1ab2801dde
56 changed files with 500 additions and 558 deletions
@@ -16,19 +16,6 @@ def add_attention_backend(server_args, attention_config):
server_args.extend(["--attention-backend", attention_config["backend"]])
@pytest.fixture(scope="module")
def rocm_aiter_fa_attention():
"""Return attention config for transcription/translation tests on ROCm.
On ROCm, audio tests require ROCM_AITER_FA attention backend.
"""
from vllm.platforms import current_platform
if current_platform.is_rocm():
return {"backend": "ROCM_AITER_FA"}
return None
@pytest.fixture
def mary_had_lamb():
path = AudioAsset("mary_had_lamb").get_local_path()
@@ -10,15 +10,13 @@ import pybase64 as base64
import pytest
import websockets
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
from vllm.assets.audio import AudioAsset
from vllm.multimodal.media.audio import load_audio
# Increase engine iteration timeout for ROCm where first-use JIT compilation
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
REALTIME_ENV_OVERRIDES = {
**ROCM_ENV_OVERRIDES,
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
}
@@ -72,17 +70,13 @@ def mary_had_lamb_audio_chunks() -> list[str]:
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_multi_chunk_streaming(
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
):
async def test_multi_chunk_streaming(model_name, mary_had_lamb_audio_chunks):
"""Test streaming multiple audio chunks before committing."""
server_args = ["--enforce-eager", "--max-model-len", "2048"]
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
@@ -174,7 +168,7 @@ async def test_multi_chunk_streaming(
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_empty_commit_does_not_crash_engine(
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
model_name, mary_had_lamb_audio_chunks
):
"""Test that committing without audio does not crash the engine.
@@ -189,8 +183,6 @@ async def test_empty_commit_does_not_crash_engine(
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
@@ -273,17 +265,13 @@ async def test_empty_commit_does_not_crash_engine(
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_session_update_invalid_model_returns_error(
model_name, rocm_aiter_fa_attention
):
async def test_session_update_invalid_model_returns_error(model_name):
"""Test that session.update with an invalid model returns an error."""
server_args = ["--enforce-eager", "--max-model-len", "2048"]
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
@@ -305,9 +293,7 @@ async def test_session_update_invalid_model_returns_error(
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
async def test_commit_without_session_update_returns_error(
model_name, rocm_aiter_fa_attention
):
async def test_commit_without_session_update_returns_error(model_name):
"""Test that committing before validating the model returns an error
and does not fall through to processing."""
server_args = ["--enforce-eager", "--max-model-len", "2048"]
@@ -315,8 +301,6 @@ async def test_commit_without_session_update_returns_error(
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
) as remote_server:
@@ -6,8 +6,7 @@ import json
import pytest
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
MISTRAL_FORMAT_ARGS = [
"--tokenizer_mode",
@@ -66,18 +65,14 @@ async def transcribe_and_check(
@pytest.mark.parametrize(
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
)
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
async def test_basic_audio(mary_had_lamb, model_name):
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
if model_name.startswith("mistralai"):
server_args += MISTRAL_FORMAT_ARGS
add_attention_backend(server_args, rocm_aiter_fa_attention)
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
with RemoteOpenAIServer(
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
client,
@@ -90,7 +85,7 @@ async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
@pytest.mark.asyncio
async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
async def test_basic_audio_with_lora(mary_had_lamb):
"""Ensure STT (transcribe) requests can pass LoRA through to generate."""
# ROCm SPECIFIC CONFIGURATION:
# To ensure the test passes on ROCm, we modify the max model length to 512.
@@ -112,12 +107,8 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
"1",
]
add_attention_backend(server_args, rocm_aiter_fa_attention)
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
with RemoteOpenAIServer(
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
) as remote_server:
with RemoteOpenAIServer(model_name, server_args) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
client,
@@ -133,18 +124,15 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
@pytest.mark.parametrize(
"model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]
)
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
async def test_basic_audio_foscolo(foscolo, model_name):
# Gemma accuracy on some of the audio samples we use is particularly bad,
# hence we use a different one here. WER is evaluated separately.
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
add_attention_backend(server_args, rocm_aiter_fa_attention)
with RemoteOpenAIServer(
model_name,
server_args,
max_wait_seconds=480,
env_dict=ROCM_ENV_OVERRIDES,
) as remote_server:
client = remote_server.get_async_client()
await transcribe_and_check(
@@ -28,27 +28,33 @@ def _get_rocm_attention_config(model_name):
Whisper uses cross-attention (ENCODER_DECODER) which ROCM_AITER_FA does
not support. For Whisper we use ROCM_AITER_UNIFIED_ATTN (or TRITON_ATTN
as fallback); other models can use ROCM_AITER_FA.
as fallback). Other models use ROCM_AITER_FA only on supported CDNA3+
hardware and otherwise use the platform default.
"""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return None
if "whisper" in model_name.lower():
try:
from vllm.platforms.rocm import get_cdna_version
try:
from vllm.platforms.rocm import get_cdna_version
if get_cdna_version() > 2:
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
except ImportError:
logger.warning(
"Could not check cdna version from rocm platform, "
"falling back to TRITON_ATTN for Whisper."
)
cdna_version = get_cdna_version()
except ImportError:
logger.warning(
"Could not check cdna version from rocm platform; "
"using a compatible fallback attention backend."
)
cdna_version = -1
if "whisper" in model_name.lower():
if cdna_version > 2:
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
return {"backend": "TRITON_ATTN"}
return {"backend": "ROCM_AITER_FA"}
if cdna_version > 2:
return {"backend": "ROCM_AITER_FA"}
return None
def _get_server_args(attention_config):