[Bugfix] Fix FlashInfer KV sharing with omitted K/V (#55864)
Signed-off-by: zhicheng Liu <lzc021026@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fe4d4109c2
commit
fbf51c7026
@@ -246,16 +246,8 @@ class Gemma4MTPAttention(nn.Module):
|
||||
|
||||
q, _ = self.rotary_emb(positions, q, None)
|
||||
|
||||
# Attention reads K/V from the target's cache via KV sharing;
|
||||
# these dummy tensors are never consumed but required by the API.
|
||||
num_tokens = q.shape[0]
|
||||
kv_dummy = torch.empty(
|
||||
num_tokens,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
attn_output = self.attn(q, kv_dummy, kv_dummy)
|
||||
# Attention reads K/V from the target's cache via KV sharing.
|
||||
attn_output = self.attn(q, None, None)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
Reference in New Issue
Block a user