[Bugfix] Fix FlashInfer KV sharing with omitted K/V (#55864)
Signed-off-by: zhicheng Liu <lzc021026@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fe4d4109c2
commit
fbf51c7026
@@ -246,16 +246,8 @@ class Gemma4MTPAttention(nn.Module):
|
||||
|
||||
q, _ = self.rotary_emb(positions, q, None)
|
||||
|
||||
# Attention reads K/V from the target's cache via KV sharing;
|
||||
# these dummy tensors are never consumed but required by the API.
|
||||
num_tokens = q.shape[0]
|
||||
kv_dummy = torch.empty(
|
||||
num_tokens,
|
||||
self.num_kv_heads * self.head_dim,
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
attn_output = self.attn(q, kv_dummy, kv_dummy)
|
||||
# Attention reads K/V from the target's cache via KV sharing.
|
||||
attn_output = self.attn(q, None, None)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
@@ -1962,8 +1962,8 @@ class FlashInferImpl(AttentionImpl):
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashInferMetadata,
|
||||
output: torch.Tensor,
|
||||
@@ -1974,8 +1974,10 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
Args:
|
||||
query: shape = [num_tokens, num_heads, head_size]
|
||||
key: shape = [num_tokens, num_kv_heads, head_size]
|
||||
value: shape = [num_tokens, num_kv_heads, head_size]
|
||||
key: shape = [num_tokens, num_kv_heads, head_size], or None for a
|
||||
KV-sharing decoder layer.
|
||||
value: shape = [num_tokens, num_kv_heads, head_size], or None for a
|
||||
KV-sharing decoder layer.
|
||||
kv_cache: [num_blocks, num_kv_heads, block_size, 2*head_size]
|
||||
attn_metadata: Metadata for attention.
|
||||
Returns:
|
||||
@@ -2066,8 +2068,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
# Inputs and outputs may be padded for CUDA graphs
|
||||
query = query[:num_actual_tokens]
|
||||
key = key[:num_actual_tokens]
|
||||
value = value[:num_actual_tokens]
|
||||
output_padded = output
|
||||
output = output[:num_actual_tokens]
|
||||
|
||||
@@ -2158,6 +2158,10 @@ class FlashInferImpl(AttentionImpl):
|
||||
prefill_wrapper = attn_metadata.prefill.wrapper
|
||||
assert prefill_wrapper is not None
|
||||
if use_dcp:
|
||||
if key is None or value is None:
|
||||
raise NotImplementedError(
|
||||
"FlashInfer DCP prefill does not support KV-sharing layers"
|
||||
)
|
||||
assert isinstance(prefill_wrapper, BatchDCPPrefillWrapper)
|
||||
assert prefill_wrapper._context._window_left == self.window_left
|
||||
assert prefill_wrapper._context._logits_soft_cap == (
|
||||
@@ -2176,8 +2180,8 @@ class FlashInferImpl(AttentionImpl):
|
||||
layer,
|
||||
prefill_query,
|
||||
kv_cache_tuple,
|
||||
key[num_decode_tokens:],
|
||||
value[num_decode_tokens:],
|
||||
key[num_decode_tokens:num_actual_tokens],
|
||||
value[num_decode_tokens:num_actual_tokens],
|
||||
out=output[num_decode_tokens:],
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user