[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f2f31b82e
commit
1ab2801dde
@@ -16,19 +16,6 @@ def add_attention_backend(server_args, attention_config):
|
||||
server_args.extend(["--attention-backend", attention_config["backend"]])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rocm_aiter_fa_attention():
|
||||
"""Return attention config for transcription/translation tests on ROCm.
|
||||
|
||||
On ROCm, audio tests require ROCM_AITER_FA attention backend.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mary_had_lamb():
|
||||
path = AudioAsset("mary_had_lamb").get_local_path()
|
||||
|
||||
@@ -10,15 +10,13 @@ import pybase64 as base64
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
# Increase engine iteration timeout for ROCm where first-use JIT compilation
|
||||
# can exceed the default 60s, causing a silent deadlock in feed_tokens.
|
||||
REALTIME_ENV_OVERRIDES = {
|
||||
**ROCM_ENV_OVERRIDES,
|
||||
"VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
|
||||
}
|
||||
|
||||
@@ -72,17 +70,13 @@ def mary_had_lamb_audio_chunks() -> list[str]:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_multi_chunk_streaming(
|
||||
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
|
||||
):
|
||||
async def test_multi_chunk_streaming(model_name, mary_had_lamb_audio_chunks):
|
||||
"""Test streaming multiple audio chunks before committing."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
@@ -174,7 +168,7 @@ async def test_multi_chunk_streaming(
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_empty_commit_does_not_crash_engine(
|
||||
model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
|
||||
model_name, mary_had_lamb_audio_chunks
|
||||
):
|
||||
"""Test that committing without audio does not crash the engine.
|
||||
|
||||
@@ -189,8 +183,6 @@ async def test_empty_commit_does_not_crash_engine(
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
@@ -273,17 +265,13 @@ async def test_empty_commit_does_not_crash_engine(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_session_update_invalid_model_returns_error(
|
||||
model_name, rocm_aiter_fa_attention
|
||||
):
|
||||
async def test_session_update_invalid_model_returns_error(model_name):
|
||||
"""Test that session.update with an invalid model returns an error."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
@@ -305,9 +293,7 @@ async def test_session_update_invalid_model_returns_error(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
async def test_commit_without_session_update_returns_error(
|
||||
model_name, rocm_aiter_fa_attention
|
||||
):
|
||||
async def test_commit_without_session_update_returns_error(model_name):
|
||||
"""Test that committing before validating the model returns an error
|
||||
and does not fall through to processing."""
|
||||
server_args = ["--enforce-eager", "--max-model-len", "2048"]
|
||||
@@ -315,8 +301,6 @@ async def test_commit_without_session_update_returns_error(
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
|
||||
@@ -6,8 +6,7 @@ import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.speech_to_text.conftest import add_attention_backend
|
||||
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer
|
||||
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
@@ -66,18 +65,14 @@ async def transcribe_and_check(
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
|
||||
async def test_basic_audio(mary_had_lamb, model_name):
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
if model_name.startswith("mistralai"):
|
||||
server_args += MISTRAL_FORMAT_ARGS
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
@@ -90,7 +85,7 @@ async def test_basic_audio(mary_had_lamb, model_name, rocm_aiter_fa_attention):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
|
||||
async def test_basic_audio_with_lora(mary_had_lamb):
|
||||
"""Ensure STT (transcribe) requests can pass LoRA through to generate."""
|
||||
# ROCm SPECIFIC CONFIGURATION:
|
||||
# To ensure the test passes on ROCm, we modify the max model length to 512.
|
||||
@@ -112,12 +107,8 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
|
||||
"1",
|
||||
]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
# Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(model_name, server_args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
client,
|
||||
@@ -133,18 +124,15 @@ async def test_basic_audio_with_lora(mary_had_lamb, rocm_aiter_fa_attention):
|
||||
@pytest.mark.parametrize(
|
||||
"model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"]
|
||||
)
|
||||
async def test_basic_audio_foscolo(foscolo, rocm_aiter_fa_attention, model_name):
|
||||
async def test_basic_audio_foscolo(foscolo, model_name):
|
||||
# Gemma accuracy on some of the audio samples we use is particularly bad,
|
||||
# hence we use a different one here. WER is evaluated separately.
|
||||
server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS]
|
||||
|
||||
add_attention_backend(server_args, rocm_aiter_fa_attention)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
max_wait_seconds=480,
|
||||
env_dict=ROCM_ENV_OVERRIDES,
|
||||
) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
await transcribe_and_check(
|
||||
|
||||
@@ -28,27 +28,33 @@ def _get_rocm_attention_config(model_name):
|
||||
|
||||
Whisper uses cross-attention (ENCODER_DECODER) which ROCM_AITER_FA does
|
||||
not support. For Whisper we use ROCM_AITER_UNIFIED_ATTN (or TRITON_ATTN
|
||||
as fallback); other models can use ROCM_AITER_FA.
|
||||
as fallback). Other models use ROCM_AITER_FA only on supported CDNA3+
|
||||
hardware and otherwise use the platform default.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return None
|
||||
|
||||
if "whisper" in model_name.lower():
|
||||
try:
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
try:
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
if get_cdna_version() > 2:
|
||||
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Could not check cdna version from rocm platform, "
|
||||
"falling back to TRITON_ATTN for Whisper."
|
||||
)
|
||||
cdna_version = get_cdna_version()
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Could not check cdna version from rocm platform; "
|
||||
"using a compatible fallback attention backend."
|
||||
)
|
||||
cdna_version = -1
|
||||
|
||||
if "whisper" in model_name.lower():
|
||||
if cdna_version > 2:
|
||||
return {"backend": "ROCM_AITER_UNIFIED_ATTN"}
|
||||
return {"backend": "TRITON_ATTN"}
|
||||
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
if cdna_version > 2:
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
return None
|
||||
|
||||
|
||||
def _get_server_args(attention_config):
|
||||
|
||||
Reference in New Issue
Block a user