From f63dca68385c41148f386c86eac796c83d5ddd3b Mon Sep 17 00:00:00 2001 From: djramic Date: Fri, 3 Jul 2026 20:53:29 +0200 Subject: [PATCH] [ROCm] Fix encoder-decoder cross-attention KV layout aliasing (#47035) Signed-off-by: Djordje Ramic Co-authored-by: Andreas Karatzas --- .../test_transcription_validation_whisper.py | 3 +- .../backends/rocm_aiter_unified_attn.py | 97 +++++++++++-------- vllm/v1/worker/gpu_model_runner.py | 34 ++++++- 3 files changed, 91 insertions(+), 43 deletions(-) diff --git a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py index 511179f7fc..bbfde877b9 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py @@ -68,7 +68,8 @@ def server(request): if request.param is not None: args += ["--attention-backend", request.param] if "AITER" in request.param: - env_dict = _AITER_ENV + # TODO: re-enable once AITER reenables fp16 unified attention. + pytest.skip("ROCM_AITER_UNIFIED_ATTN does not support fp16") with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: yield remote_server diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index d8363169a8..66d57917fa 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -148,26 +148,10 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): def _split_kv_cache( self, kv_cache: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: - if self.attn_type != AttentionType.ENCODER_DECODER: - return kv_cache.unbind(1) - - # NOTE: Encoder-decoder layers can share the same raw KV allocation with - # ROCM_ATTN decoder layers, whose physical layout is K/V first. Keep - # this cross-attention path on that physical layout so block IDs do not - # alias different bytes across the shared allocation. - num_blocks, _, block_size, num_kv_heads, head_size = kv_cache.shape - block_stride = block_size * num_kv_heads * head_size - kv_cache = kv_cache.as_strided( - (2, num_blocks, block_size, num_kv_heads, head_size), - ( - num_blocks * block_stride, - block_stride, - num_kv_heads * head_size, - head_size, - 1, - ), - ) - return kv_cache.unbind(0) + # Blocks-first ``(num_blocks, 2, ...)``. The model runner normalizes any + # shared decoder/cross-attention allocation to this layout, so no + # per-backend restriding is needed here. + return kv_cache.unbind(1) def forward( self, @@ -242,27 +226,58 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): max_seqlen_k = attn_metadata.max_seq_len block_table = attn_metadata.block_table - self.unified_attention( - q=query[:num_actual_tokens], - k=key_cache, - v=value_cache, - out=output[:num_actual_tokens], - cu_seqlens_q=cu_seqlens_q, - max_seqlen_q=max_seqlen_q, - seqused_k=seqused_k, - max_seqlen_k=max_seqlen_k, - softmax_scale=softmax_scale, - causal=True, - alibi_slopes=self.alibi_slopes, - window_size=self.sliding_window, - block_table=block_table, - softcap=self.logits_soft_cap, - q_descale=layer._q_scale if query.dtype == self.fp8_dtype else None, - k_descale=layer._k_scale, - v_descale=layer._v_scale, - sinks=self.sinks, - output_scale=output_scale, - ) + if attn_metadata.causal: + self.unified_attention( + q=query[:num_actual_tokens], + k=key_cache, + v=value_cache, + out=output[:num_actual_tokens], + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=max_seqlen_q, + seqused_k=seqused_k, + max_seqlen_k=max_seqlen_k, + softmax_scale=softmax_scale, + causal=True, + alibi_slopes=self.alibi_slopes, + window_size=self.sliding_window, + block_table=block_table, + softcap=self.logits_soft_cap, + q_descale=layer._q_scale if query.dtype == self.fp8_dtype else None, + k_descale=layer._k_scale, + v_descale=layer._v_scale, + sinks=self.sinks, + output_scale=output_scale, + ) + else: + # The aiter kernel is causal-only. Non-causal cross-attention + # (ENCODER_DECODER, e.g. Whisper) falls back to the vLLM Triton + # unified kernel, which shares this layout and honors the flag. + from vllm.v1.attention.ops.triton_unified_attention import ( + unified_attention as triton_unified_attention, + ) + + descale_shape = (cu_seqlens_q.shape[0] - 1, key_cache.shape[2]) + triton_unified_attention( + q=query[:num_actual_tokens], + k=key_cache, + v=value_cache, + out=output[:num_actual_tokens], + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=max_seqlen_q, + seqused_k=seqused_k, + max_seqlen_k=max_seqlen_k, + softmax_scale=softmax_scale, + causal=attn_metadata.causal, + alibi_slopes=self.alibi_slopes, + window_size=self.sliding_window, + block_table=block_table, + softcap=self.logits_soft_cap, + q_descale=layer._q_scale if query.dtype == self.fp8_dtype else None, + k_descale=layer._k_scale.expand(descale_shape), + v_descale=layer._v_scale.expand(descale_shape), + sinks=self.sinks, + output_scale=output_scale, + ) return output diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index d67a055259..53795951cb 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -7196,11 +7196,43 @@ class GPUModelRunner( else: raise NotImplementedError - if has_attn and has_mamba: + # Reconcile divergent KV layouts to blocks-first. Triggered by hybrid + # attention/mamba models, and by encoder-decoder models whose shared + # decoder/cross-attention allocation mixes K/V-first and blocks-first + # backends (see _has_mixed_attention_kv_layout). + if has_attn and ( + has_mamba or self._has_mixed_attention_kv_layout(kernel_block_sizes) + ): self._update_hybrid_attention_mamba_layout(kv_caches, kernel_block_sizes) return kv_caches + def _has_mixed_attention_kv_layout(self, kernel_block_sizes: list[int]) -> bool: + """Whether attention groups disagree on the physical KV cache layout. + + Encoder-decoder models (e.g. Whisper) share one raw KV allocation + between a decoder self-attention layer (K/V-first ROCM_ATTN, block dim + 1) and a cross-attention layer (blocks-first, block dim 0). Mixed block + dims mean a block ID maps to different bytes per layer, so the shared + buffer must be normalized to a single (blocks-first) layout. + """ + block_dims: set[int] = set() + for group in self._kv_cache_spec_attn_group_iterator(): + kv_cache_spec = group.kv_cache_spec + if not isinstance(kv_cache_spec, AttentionSpec): + continue + if group.kv_cache_group_id == len(kernel_block_sizes): + continue + block_dims.add( + group.backend.get_kv_cache_block_dim( + kernel_block_sizes[group.kv_cache_group_id], + kv_cache_spec.num_kv_heads, + kv_cache_spec.head_size, + cache_dtype_str=self.cache_config.cache_dtype, + ) + ) + return len(block_dims) > 1 + def _update_hybrid_attention_mamba_layout( self, kv_caches: dict[str, torch.Tensor], kernel_block_sizes: list[int] ) -> None: