[Bugfix] Fix FlashInfer KV sharing with omitted K/V (#55864)

Signed-off-by: zhicheng Liu <lzc021026@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zhicheng Liu
2026-09-10 23:07:52 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fe4d4109c2
commit fbf51c7026
2 changed files with 14 additions and 18 deletions
+2 -10
View File
@@ -246,16 +246,8 @@ class Gemma4MTPAttention(nn.Module):
q, _ = self.rotary_emb(positions, q, None)
# Attention reads K/V from the target's cache via KV sharing;
# these dummy tensors are never consumed but required by the API.
num_tokens = q.shape[0]
kv_dummy = torch.empty(
num_tokens,
self.num_kv_heads * self.head_dim,
dtype=q.dtype,
device=q.device,
)
attn_output = self.attn(q, kv_dummy, kv_dummy)
# Attention reads K/V from the target's cache via KV sharing.
attn_output = self.attn(q, None, None)
output, _ = self.o_proj(attn_output)
return output