From fbf51c70269bb0d8333e14c5788e628e33c72f5d Mon Sep 17 00:00:00 2001 From: Zhicheng Liu <2084597944@qq.com> Date: Fri, 11 Sep 2026 14:07:52 +0800 Subject: [PATCH] [Bugfix] Fix FlashInfer KV sharing with omitted K/V (#55864) Signed-off-by: zhicheng Liu Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/gemma4_mtp.py | 12 ++---------- vllm/v1/attention/backends/flashinfer.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/vllm/model_executor/models/gemma4_mtp.py b/vllm/model_executor/models/gemma4_mtp.py index 5e25a80d6b..639af0ea29 100644 --- a/vllm/model_executor/models/gemma4_mtp.py +++ b/vllm/model_executor/models/gemma4_mtp.py @@ -246,16 +246,8 @@ class Gemma4MTPAttention(nn.Module): q, _ = self.rotary_emb(positions, q, None) - # Attention reads K/V from the target's cache via KV sharing; - # these dummy tensors are never consumed but required by the API. - num_tokens = q.shape[0] - kv_dummy = torch.empty( - num_tokens, - self.num_kv_heads * self.head_dim, - dtype=q.dtype, - device=q.device, - ) - attn_output = self.attn(q, kv_dummy, kv_dummy) + # Attention reads K/V from the target's cache via KV sharing. + attn_output = self.attn(q, None, None) output, _ = self.o_proj(attn_output) return output diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 4b79135cab..3547a45c98 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1962,8 +1962,8 @@ class FlashInferImpl(AttentionImpl): self, layer: torch.nn.Module, query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + key: torch.Tensor | None, + value: torch.Tensor | None, kv_cache: torch.Tensor, attn_metadata: FlashInferMetadata, output: torch.Tensor, @@ -1974,8 +1974,10 @@ class FlashInferImpl(AttentionImpl): Args: query: shape = [num_tokens, num_heads, head_size] - key: shape = [num_tokens, num_kv_heads, head_size] - value: shape = [num_tokens, num_kv_heads, head_size] + key: shape = [num_tokens, num_kv_heads, head_size], or None for a + KV-sharing decoder layer. + value: shape = [num_tokens, num_kv_heads, head_size], or None for a + KV-sharing decoder layer. kv_cache: [num_blocks, num_kv_heads, block_size, 2*head_size] attn_metadata: Metadata for attention. Returns: @@ -2066,8 +2068,6 @@ class FlashInferImpl(AttentionImpl): # Inputs and outputs may be padded for CUDA graphs query = query[:num_actual_tokens] - key = key[:num_actual_tokens] - value = value[:num_actual_tokens] output_padded = output output = output[:num_actual_tokens] @@ -2158,6 +2158,10 @@ class FlashInferImpl(AttentionImpl): prefill_wrapper = attn_metadata.prefill.wrapper assert prefill_wrapper is not None if use_dcp: + if key is None or value is None: + raise NotImplementedError( + "FlashInfer DCP prefill does not support KV-sharing layers" + ) assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper) assert prefill_wrapper._context._window_left == self.window_left assert prefill_wrapper._context._logits_soft_cap == ( @@ -2176,8 +2180,8 @@ class FlashInferImpl(AttentionImpl): layer, prefill_query, kv_cache_tuple, - key[num_decode_tokens:], - value[num_decode_tokens:], + key[num_decode_tokens:num_actual_tokens], + value[num_decode_tokens:num_actual_tokens], out=output[num_decode_tokens:], ) else: