[Bugfix][DCP] Handle sparse MLA metadata after DCP Manager refactor (#52377)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -824,6 +824,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
use_mha = True
|
||||
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill = getattr(attn_metadata, "prefill", None)
|
||||
@@ -992,16 +993,21 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert lse is not None
|
||||
assert self.dcp_manager is not None
|
||||
seq_lens = (
|
||||
attn_metadata.decode.seq_lens
|
||||
if attn_metadata.decode is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
decode_metadata = getattr(attn_metadata, "decode", None)
|
||||
if not use_mha:
|
||||
seq_lens = cast(torch.Tensor, attn_metadata.seq_lens) # type: ignore[attr-defined]
|
||||
query_start_loc = attn_metadata.query_start_loc
|
||||
else:
|
||||
seq_lens = (
|
||||
decode_metadata.seq_lens
|
||||
if decode_metadata is not None
|
||||
else cast(torch.Tensor, attn_metadata.seq_lens)[ # type: ignore[attr-defined]
|
||||
: attn_metadata.num_decodes
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
)
|
||||
query_start_loc = attn_metadata.query_start_loc[
|
||||
: attn_metadata.num_decodes + 1
|
||||
]
|
||||
attn_out = self.dcp_manager.combine(
|
||||
attn_out,
|
||||
lse,
|
||||
|
||||
Reference in New Issue
Block a user