From cd947f5c629f16fb1ac8ac243481ce82bfa6f5a2 Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Tue, 29 Sep 2026 15:27:03 +0200 Subject: [PATCH] [MM] Fix compiled ViT attention output layouts (#58182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Zakrzewski Co-authored-by: OpenAI Codex Co-authored-by: Nicolò Lucchesi --- tests/v1/attention/test_vit_attn_wrappers.py | 87 ++++++++++++++++++++ vllm/v1/attention/ops/vit_attn_wrappers.py | 44 ++++++---- 2 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 tests/v1/attention/test_vit_attn_wrappers.py diff --git a/tests/v1/attention/test_vit_attn_wrappers.py b/tests/v1/attention/test_vit_attn_wrappers.py new file mode 100644 index 0000000000..5d6cb323ba --- /dev/null +++ b/tests/v1/attention/test_vit_attn_wrappers.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import sys +from types import ModuleType, SimpleNamespace + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from vllm.platforms import current_platform +from vllm.v1.attention.ops import vit_attn_wrappers + + +def _noncontiguous_bshd(device: torch.device | None = None) -> torch.Tensor: + return ( + torch.arange(120, dtype=torch.float32, device=device) + .reshape(1, 3, 5, 8) + .transpose(1, 2) + ) + + +def test_registered_attention_wrapper_fake_layouts(): + with FakeTensorMode(): + q = _noncontiguous_bshd() + cu_seqlens = torch.tensor([0, 5], dtype=torch.int32) + max_seqlen = torch.tensor(5, dtype=torch.int32) + flashinfer_cu_seqlens = torch.tensor([0, 5, 0, 5], dtype=torch.int32) + sequence_lengths = torch.tensor([5], dtype=torch.int32) + + outputs = ( + torch.ops.vllm.flash_attn_maxseqlen_wrapper( + q, q, q, 1, False, None, None, cu_seqlens, max_seqlen + ), + torch.ops.vllm.triton_attn_wrapper( + q, q, q, 1, None, cu_seqlens, max_seqlen + ), + torch.ops.vllm.torch_sdpa_wrapper(q, q, q, None, None, False), + torch.ops.vllm.flashinfer_wrapper( + q, + q, + q, + 1.0, + torch.empty(1, dtype=torch.uint8), + flashinfer_cu_seqlens, + max_seqlen, + sequence_lengths, + None, + None, + None, + None, + ), + ) + + assert all(output.is_contiguous() for output in outputs) + + +def test_aiter_flash_attn_omits_out_and_returns_contiguous_output( + monkeypatch: pytest.MonkeyPatch, +): + aiter = ModuleType("aiter") + + def flash_attn_varlen_func(q, k, v, **kwargs): + assert "out" not in kwargs + return q + + aiter.__dict__["flash_attn_varlen_func"] = flash_attn_varlen_func + monkeypatch.setitem(sys.modules, "aiter", aiter) + monkeypatch.setattr( + vit_attn_wrappers, "current_platform", SimpleNamespace(is_rocm=lambda: True) + ) + q = _noncontiguous_bshd() + assert not q.is_contiguous() + + output = vit_attn_wrappers.flash_attn_maxseqlen_wrapper(q, q, q, 1, True, None) + + assert torch.equal(output, q) + assert output.is_contiguous() + + +@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires GPU") +def test_sdpa_output_layout_without_cu_seqlens(): + q = _noncontiguous_bshd(torch.device("cuda")) + + output = vit_attn_wrappers.vit_torch_sdpa_wrapper(q, q, q) + + assert output.is_contiguous() diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index 36f7de393c..bef9ddb4d0 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -58,13 +58,15 @@ def flash_attn_maxseqlen_wrapper( max_seqlen = max_seqlen.item() q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) - - if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing(): - # workaround for encoder CUDA-graph replay paddings with NaNs edge case - kwargs["out"] = torch.zeros( - q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device - ) - + supports_out = not current_platform.is_rocm() + if supports_out: + if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing(): + # workaround for encoder CUDA-graph replay paddings with NaNs edge case + kwargs["out"] = torch.zeros( + q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device + ) + else: + kwargs["out"] = torch.empty_like(q, memory_format=torch.contiguous_format) output = flash_attn_varlen_func( q, k, @@ -78,6 +80,8 @@ def flash_attn_maxseqlen_wrapper( softmax_scale=scale, **kwargs, ) + if not supports_out: + output = output.contiguous() context_layer = einops.rearrange(output, "(b s) h d -> b s h d", b=batch_size) return context_layer @@ -93,7 +97,7 @@ def flash_attn_maxseqlen_wrapper_fake( cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -179,7 +183,7 @@ def triton_attn_wrapper( max_seqlen = max_seqlen.item() q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) - output = torch.empty_like(q) + output = torch.empty_like(q, memory_format=torch.contiguous_format) context_attention_fwd( q, k, @@ -207,7 +211,7 @@ def triton_attn_wrapper_fake( cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -281,7 +285,7 @@ def torch_sdpa_wrapper( v = v.contiguous() if cu_seqlens is None: - return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa) + return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa).contiguous() outputs = [] @@ -302,11 +306,11 @@ def torch_sdpa_wrapper_fake( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - scale: float | None, - cu_seqlens: torch.Tensor | None, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, enable_gqa: bool = False, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -368,6 +372,11 @@ def flashinfer_wrapper( with gpu_sync_allowed(): max_seqlen = max_seqlen.item() + output_buffer = torch.empty_like( + q, + dtype=o_data_type, + memory_format=torch.contiguous_format, + ) output, _ = cudnn_batch_prefill_with_kv_cache( q, k, @@ -388,6 +397,7 @@ def flashinfer_wrapper( k_scale=k_scale, v_scale=v_scale, o_data_type=o_data_type, + out=output_buffer, ) if is_reshaped: @@ -410,7 +420,11 @@ def vit_flashinfer_wrapper_fake( v_scale: torch.Tensor | None = None, o_data_type: torch.dtype | None = None, ) -> torch.Tensor: - return torch.empty_like(q, dtype=o_data_type or q.dtype) + return torch.empty_like( + q, + dtype=o_data_type, + memory_format=torch.contiguous_format, + ) direct_register_custom_op(