[Bugfix][MLA] Fix LSE log-base mismatch in DCP + FlashInfer MLA decode (#47079)
Signed-off-by: girasoley <girasoleyang@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -818,14 +818,14 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=True,
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
else:
|
||||
attn_out = cp_lse_ag_out_rs(
|
||||
attn_out,
|
||||
lse,
|
||||
get_dcp_group(),
|
||||
is_lse_base_on_e=True,
|
||||
is_lse_base_on_e=self.impl.lse_base_on_e,
|
||||
)
|
||||
|
||||
# v_up projection
|
||||
|
||||
@@ -756,6 +756,17 @@ class AttentionImplBase(ABC, Generic[T]):
|
||||
# Some features like decode context parallelism require the softmax lse.
|
||||
can_return_lse_for_decode: bool = False
|
||||
|
||||
# Base of the logarithm used by this backend when returning softmax lse.
|
||||
# True => natural log (lse = ln(sum(exp(qk))))
|
||||
# -- e.g. Triton MLA, FlashAttention, FlashMLA, Cutlass MLA
|
||||
# False => base 2 (lse = log2(sum(exp(qk))))
|
||||
# -- e.g. FlashInfer trtllm-gen MLA
|
||||
# The DCP combine kernel (cp_lse_ag_out_rs / dcp_a2a_lse_reduce in
|
||||
# vllm/v1/attention/ops/common.py) branches on this via its IS_BASE_E
|
||||
# constexpr; getting it wrong silently corrupts the cross-shard
|
||||
# softmax denominator.
|
||||
lse_base_on_e: bool = True
|
||||
|
||||
# Whether the attention impl supports Prefill Context Parallelism.
|
||||
supports_pcp: bool = False
|
||||
# Whether the attention impl(or ops) supports MTP
|
||||
|
||||
@@ -114,6 +114,13 @@ g_fi_workspace = torch.zeros(
|
||||
|
||||
class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
# trtllm-gen MLA decode emits LSE in log2 (per flashinfer's own
|
||||
# reference at flashinfer/trace/templates/attention.py:81:
|
||||
# `logsumexp / log(2.0)`). Override the AttentionImplBase default
|
||||
# so MLAAttention's DCP combine branches on the correct base
|
||||
# (IS_BASE_E=False uses tl.exp2/tl.log2 natively, avoiding an FP
|
||||
# multiply per decode step).
|
||||
lse_base_on_e: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user