[Bugfix] Fix FlashInfer KV sharing with omitted K/V (#55864)

Signed-off-by: zhicheng Liu <lzc021026@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zhicheng Liu
2026-09-10 23:07:52 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fe4d4109c2
commit fbf51c7026
2 changed files with 14 additions and 18 deletions
+2 -10
View File
@@ -246,16 +246,8 @@ class Gemma4MTPAttention(nn.Module):
q, _ = self.rotary_emb(positions, q, None)
# Attention reads K/V from the target's cache via KV sharing;
# these dummy tensors are never consumed but required by the API.
num_tokens = q.shape[0]
kv_dummy = torch.empty(
num_tokens,
self.num_kv_heads * self.head_dim,
dtype=q.dtype,
device=q.device,
)
attn_output = self.attn(q, kv_dummy, kv_dummy)
# Attention reads K/V from the target's cache via KV sharing.
attn_output = self.attn(q, None, None)
output, _ = self.o_proj(attn_output)
return output
+12 -8
View File
@@ -1962,8 +1962,8 @@ class FlashInferImpl(AttentionImpl):
self,
layer: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
kv_cache: torch.Tensor,
attn_metadata: FlashInferMetadata,
output: torch.Tensor,
@@ -1974,8 +1974,10 @@ class FlashInferImpl(AttentionImpl):
Args:
query: shape = [num_tokens, num_heads, head_size]
key: shape = [num_tokens, num_kv_heads, head_size]
value: shape = [num_tokens, num_kv_heads, head_size]
key: shape = [num_tokens, num_kv_heads, head_size], or None for a
KV-sharing decoder layer.
value: shape = [num_tokens, num_kv_heads, head_size], or None for a
KV-sharing decoder layer.
kv_cache: [num_blocks, num_kv_heads, block_size, 2*head_size]
attn_metadata: Metadata for attention.
Returns:
@@ -2066,8 +2068,6 @@ class FlashInferImpl(AttentionImpl):
# Inputs and outputs may be padded for CUDA graphs
query = query[:num_actual_tokens]
key = key[:num_actual_tokens]
value = value[:num_actual_tokens]
output_padded = output
output = output[:num_actual_tokens]
@@ -2158,6 +2158,10 @@ class FlashInferImpl(AttentionImpl):
prefill_wrapper = attn_metadata.prefill.wrapper
assert prefill_wrapper is not None
if use_dcp:
if key is None or value is None:
raise NotImplementedError(
"FlashInfer DCP prefill does not support KV-sharing layers"
)
assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper)
assert prefill_wrapper._context._window_left == self.window_left
assert prefill_wrapper._context._logits_soft_cap == (
@@ -2176,8 +2180,8 @@ class FlashInferImpl(AttentionImpl):
layer,
prefill_query,
kv_cache_tuple,
key[num_decode_tokens:],
value[num_decode_tokens:],
key[num_decode_tokens:num_actual_tokens],
value[num_decode_tokens:num_actual_tokens],
out=output[num_decode_tokens:],
)
else: