[PCP][DCP] Enable PCP+DCP on sparse-MLA models (#56157)
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
patryk.saffer
patryk.saffer
Lucas Wilkinson
Patryk Saffer
Claude Fable 5.1
parent
72d4d83157
commit
ebe1dec2da
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -121,6 +121,7 @@ class DeepseekV32Indexer(nn.Module):
|
||||
class DeepseekV32Attention(MLAAttention):
|
||||
indexer: "DeepseekV32Indexer | None"
|
||||
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
|
||||
supports_pcp_dcp: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -569,8 +570,9 @@ class DeepseekV32Attention(MLAAttention):
|
||||
seq_lens: torch.Tensor | None
|
||||
query_start_loc: torch.Tensor | None
|
||||
if self.use_pcp:
|
||||
if attn_metadata.decode is not None:
|
||||
seq_lens = attn_metadata.decode.seq_lens
|
||||
decode_metadata = getattr(attn_metadata, "decode", None)
|
||||
if decode_metadata is not None:
|
||||
seq_lens = decode_metadata.seq_lens
|
||||
else:
|
||||
all_seq_lens = cast(
|
||||
torch.Tensor,
|
||||
@@ -585,12 +587,23 @@ class DeepseekV32Attention(MLAAttention):
|
||||
# PCP-only empty-shard metadata is needed.
|
||||
seq_lens = None
|
||||
query_start_loc = None
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
# Under PCP+DCP the prefill rows attended over the gathered KV, so
|
||||
# only the decode rows carry an LSE and take part in the merge.
|
||||
num_merge_rows = lse.shape[0]
|
||||
if num_merge_rows == attn_out.shape[0]:
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
elif num_merge_rows > 0:
|
||||
attn_out[:num_merge_rows] = self.dcp_manager.combine(
|
||||
attn_out[:num_merge_rows],
|
||||
lse,
|
||||
seq_lens=seq_lens, # type: ignore[arg-type]
|
||||
query_start_loc=query_start_loc, # type: ignore[arg-type]
|
||||
)
|
||||
if self.use_pcp:
|
||||
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user