[MM] Fix compiled ViT attention output layouts (#58182)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Nicolò Lucchesi
parent
d95d1dcfb9
commit
cd947f5c62
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops import vit_attn_wrappers
|
||||
|
||||
|
||||
def _noncontiguous_bshd(device: torch.device | None = None) -> torch.Tensor:
|
||||
return (
|
||||
torch.arange(120, dtype=torch.float32, device=device)
|
||||
.reshape(1, 3, 5, 8)
|
||||
.transpose(1, 2)
|
||||
)
|
||||
|
||||
|
||||
def test_registered_attention_wrapper_fake_layouts():
|
||||
with FakeTensorMode():
|
||||
q = _noncontiguous_bshd()
|
||||
cu_seqlens = torch.tensor([0, 5], dtype=torch.int32)
|
||||
max_seqlen = torch.tensor(5, dtype=torch.int32)
|
||||
flashinfer_cu_seqlens = torch.tensor([0, 5, 0, 5], dtype=torch.int32)
|
||||
sequence_lengths = torch.tensor([5], dtype=torch.int32)
|
||||
|
||||
outputs = (
|
||||
torch.ops.vllm.flash_attn_maxseqlen_wrapper(
|
||||
q, q, q, 1, False, None, None, cu_seqlens, max_seqlen
|
||||
),
|
||||
torch.ops.vllm.triton_attn_wrapper(
|
||||
q, q, q, 1, None, cu_seqlens, max_seqlen
|
||||
),
|
||||
torch.ops.vllm.torch_sdpa_wrapper(q, q, q, None, None, False),
|
||||
torch.ops.vllm.flashinfer_wrapper(
|
||||
q,
|
||||
q,
|
||||
q,
|
||||
1.0,
|
||||
torch.empty(1, dtype=torch.uint8),
|
||||
flashinfer_cu_seqlens,
|
||||
max_seqlen,
|
||||
sequence_lengths,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
)
|
||||
|
||||
assert all(output.is_contiguous() for output in outputs)
|
||||
|
||||
|
||||
def test_aiter_flash_attn_omits_out_and_returns_contiguous_output(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
aiter = ModuleType("aiter")
|
||||
|
||||
def flash_attn_varlen_func(q, k, v, **kwargs):
|
||||
assert "out" not in kwargs
|
||||
return q
|
||||
|
||||
aiter.__dict__["flash_attn_varlen_func"] = flash_attn_varlen_func
|
||||
monkeypatch.setitem(sys.modules, "aiter", aiter)
|
||||
monkeypatch.setattr(
|
||||
vit_attn_wrappers, "current_platform", SimpleNamespace(is_rocm=lambda: True)
|
||||
)
|
||||
q = _noncontiguous_bshd()
|
||||
assert not q.is_contiguous()
|
||||
|
||||
output = vit_attn_wrappers.flash_attn_maxseqlen_wrapper(q, q, q, 1, True, None)
|
||||
|
||||
assert torch.equal(output, q)
|
||||
assert output.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires GPU")
|
||||
def test_sdpa_output_layout_without_cu_seqlens():
|
||||
q = _noncontiguous_bshd(torch.device("cuda"))
|
||||
|
||||
output = vit_attn_wrappers.vit_torch_sdpa_wrapper(q, q, q)
|
||||
|
||||
assert output.is_contiguous()
|
||||
@@ -58,13 +58,15 @@ def flash_attn_maxseqlen_wrapper(
|
||||
max_seqlen = max_seqlen.item()
|
||||
|
||||
q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
|
||||
|
||||
if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing():
|
||||
# workaround for encoder CUDA-graph replay paddings with NaNs edge case
|
||||
kwargs["out"] = torch.zeros(
|
||||
q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device
|
||||
)
|
||||
|
||||
supports_out = not current_platform.is_rocm()
|
||||
if supports_out:
|
||||
if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing():
|
||||
# workaround for encoder CUDA-graph replay paddings with NaNs edge case
|
||||
kwargs["out"] = torch.zeros(
|
||||
q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device
|
||||
)
|
||||
else:
|
||||
kwargs["out"] = torch.empty_like(q, memory_format=torch.contiguous_format)
|
||||
output = flash_attn_varlen_func(
|
||||
q,
|
||||
k,
|
||||
@@ -78,6 +80,8 @@ def flash_attn_maxseqlen_wrapper(
|
||||
softmax_scale=scale,
|
||||
**kwargs,
|
||||
)
|
||||
if not supports_out:
|
||||
output = output.contiguous()
|
||||
context_layer = einops.rearrange(output, "(b s) h d -> b s h d", b=batch_size)
|
||||
return context_layer
|
||||
|
||||
@@ -93,7 +97,7 @@ def flash_attn_maxseqlen_wrapper_fake(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(q)
|
||||
return torch.empty_like(q, memory_format=torch.contiguous_format)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -179,7 +183,7 @@ def triton_attn_wrapper(
|
||||
max_seqlen = max_seqlen.item()
|
||||
|
||||
q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
|
||||
output = torch.empty_like(q)
|
||||
output = torch.empty_like(q, memory_format=torch.contiguous_format)
|
||||
context_attention_fwd(
|
||||
q,
|
||||
k,
|
||||
@@ -207,7 +211,7 @@ def triton_attn_wrapper_fake(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(q)
|
||||
return torch.empty_like(q, memory_format=torch.contiguous_format)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -281,7 +285,7 @@ def torch_sdpa_wrapper(
|
||||
v = v.contiguous()
|
||||
|
||||
if cu_seqlens is None:
|
||||
return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa)
|
||||
return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa).contiguous()
|
||||
|
||||
outputs = []
|
||||
|
||||
@@ -302,11 +306,11 @@ def torch_sdpa_wrapper_fake(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
scale: float | None,
|
||||
cu_seqlens: torch.Tensor | None,
|
||||
scale: float | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
enable_gqa: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(q)
|
||||
return torch.empty_like(q, memory_format=torch.contiguous_format)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -368,6 +372,11 @@ def flashinfer_wrapper(
|
||||
with gpu_sync_allowed():
|
||||
max_seqlen = max_seqlen.item()
|
||||
|
||||
output_buffer = torch.empty_like(
|
||||
q,
|
||||
dtype=o_data_type,
|
||||
memory_format=torch.contiguous_format,
|
||||
)
|
||||
output, _ = cudnn_batch_prefill_with_kv_cache(
|
||||
q,
|
||||
k,
|
||||
@@ -388,6 +397,7 @@ def flashinfer_wrapper(
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
o_data_type=o_data_type,
|
||||
out=output_buffer,
|
||||
)
|
||||
|
||||
if is_reshaped:
|
||||
@@ -410,7 +420,11 @@ def vit_flashinfer_wrapper_fake(
|
||||
v_scale: torch.Tensor | None = None,
|
||||
o_data_type: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(q, dtype=o_data_type or q.dtype)
|
||||
return torch.empty_like(
|
||||
q,
|
||||
dtype=o_data_type,
|
||||
memory_format=torch.contiguous_format,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
|
||||
Reference in New Issue
Block a user