[ROCm] Remove stale SDPA and skinny GEMM workarounds (#50907)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f2f31b82e
commit
1ab2801dde
@@ -134,12 +134,16 @@ def rocm_unquantized_gemm_impl(
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# Given the above, how many CUs would we need?
|
||||
CuNeeded = rndup_cus * GrpsShrB
|
||||
# candidate for atomic reduce count splitk?
|
||||
# Deterministic reduction stores one float workspace value per K shard.
|
||||
fits_wvsplitkrc = (
|
||||
N_p2 * m * ((k + 512 - 1) // 512)
|
||||
) <= 128 * 1024 * 12 # deterministic
|
||||
fits_wvsplitkrc &= CuNeeded <= cu_count
|
||||
|
||||
skinny_operands_compatible = weight.is_contiguous() and (
|
||||
bias is None or bias.is_contiguous()
|
||||
)
|
||||
|
||||
use_skinny_reduce_counting = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and on_gfx950()
|
||||
@@ -151,12 +155,13 @@ def rocm_unquantized_gemm_impl(
|
||||
and k > 512
|
||||
and m % 16 == 0
|
||||
and fits_wvsplitkrc
|
||||
and weight.is_contiguous()
|
||||
and skinny_operands_compatible
|
||||
)
|
||||
)
|
||||
|
||||
if use_skinny_reduce_counting:
|
||||
return ops.wvSplitKrc(x, weight, cu_count, bias)
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
|
||||
|
||||
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
|
||||
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
|
||||
@@ -174,10 +179,13 @@ def rocm_unquantized_gemm_impl(
|
||||
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and k % 8 == 0
|
||||
and skinny_operands_compatible
|
||||
)
|
||||
|
||||
if use_skinny:
|
||||
x_view = x.reshape(-1, x.size(-1))
|
||||
# The skinny kernels assume contiguous K elements. A shape-preserving
|
||||
# reshape can retain a transposed activation's non-contiguous strides.
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
if m > 8 and 0 < n <= 5:
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
@@ -53,7 +53,6 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -727,15 +726,9 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_embeddings = audio_output.pooler_output
|
||||
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
@@ -756,23 +749,7 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
# ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
|
||||
# with flash_sdp and mem_efficient_sdp
|
||||
# TODO: [ROCm] Fix accuracy issues with flash backend
|
||||
logger.debug(
|
||||
"ROCm platform detected. Forcing math SDP backend "
|
||||
"for vision encoder. Currently ROCm platform has "
|
||||
"accuracy issues with `flash_sdp` and"
|
||||
"`mem_efficient_sdp` backends. See issue: "
|
||||
"https://github.com/vllm-project/vllm/issues/30167"
|
||||
)
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
|
||||
Reference in New Issue
Block a user