[Attention][MLA] FlashMLA sparse: DCP on the fp8_ds_mla mixed-batch path + MTP (#46514)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3130573630
commit
63ff748f65
@@ -1492,6 +1492,8 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
|
||||
topk_indices_buffer=topk_indices,
|
||||
num_heads=2,
|
||||
kv_lora_rank=1,
|
||||
dcp_world_size=1,
|
||||
need_to_return_lse_for_decode=False,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
impl._forward_fp8_kv_mixed_batch = MethodType(
|
||||
@@ -1512,3 +1514,155 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
|
||||
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
|
||||
assert output.shape == (num_decode_tokens, 2, 1)
|
||||
assert lse is None
|
||||
|
||||
|
||||
def _build_sparse_dcp_vllm_config(
|
||||
local_heads: int,
|
||||
dcp_world_size: int,
|
||||
comm_backend: str = "ag_rs",
|
||||
):
|
||||
"""Minimal sparse-MLA VllmConfig for the FlashMLASparse DCP head-envelope
|
||||
guard. TP is simulated by mocking ``get_num_attention_heads`` to return the
|
||||
per-rank head count, as the decode-correctness test above does.
|
||||
"""
|
||||
kv_lora_rank = 512
|
||||
qk_nope_head_dim = 128
|
||||
qk_rope_head_dim = 64
|
||||
v_head_dim = 128
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
topk_tokens = 128
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=4096,
|
||||
block_size=64,
|
||||
hf_config_override={
|
||||
"index_topk": topk_tokens,
|
||||
"attn_module_list_cfg": [{"topk_tokens": topk_tokens}],
|
||||
},
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.dtype = torch.bfloat16
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: local_heads, model_config
|
||||
)
|
||||
model_config.get_num_kv_heads = MethodType(
|
||||
lambda self, parallel_config: 1, model_config
|
||||
)
|
||||
model_config.get_head_size = MethodType(lambda self: head_size, model_config)
|
||||
model_config.get_sliding_window = MethodType(lambda self: None, model_config)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = "fp8_ds_mla"
|
||||
vllm_config.parallel_config.decode_context_parallel_size = dcp_world_size
|
||||
vllm_config.parallel_config.dcp_comm_backend = comm_backend
|
||||
# The base builder clones the layer's dense-MHA prefill backend from
|
||||
# static_forward_context; the guard tests never run prefill.
|
||||
vllm_config.compilation_config.static_forward_context["placeholder"] = (
|
||||
SimpleNamespace(prefill_backend=None)
|
||||
)
|
||||
return vllm_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"local_heads,dcp_world_size,should_raise",
|
||||
[
|
||||
(16, 8, True),
|
||||
(24, 4, True),
|
||||
(16, 4, False),
|
||||
(16, 1, False),
|
||||
],
|
||||
)
|
||||
def test_fp8_dcp_head_envelope_guard(local_heads, dcp_world_size, should_raise):
|
||||
"""The fp8 decode envelope (head padding + tile-scheduler metadata) is
|
||||
sized from the local head count while the kernel runs on the DCP-gathered
|
||||
heads, so the builder must reject configs where the two pad differently.
|
||||
"""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
vllm_config = _build_sparse_dcp_vllm_config(local_heads, dcp_world_size)
|
||||
kv_cache_spec = create_standard_kv_cache_spec(vllm_config)
|
||||
builder_cls = FlashMLASparseBackend.get_builder_cls()
|
||||
|
||||
if should_raise:
|
||||
with pytest.raises(NotImplementedError, match="envelope"):
|
||||
builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
else:
|
||||
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
|
||||
gathered_heads = local_heads * dcp_world_size
|
||||
local_pad = 64 if local_heads <= 64 else 128
|
||||
gathered_pad = 64 if gathered_heads <= 64 else 128
|
||||
assert builder.fp8_decode_padded_heads == local_pad
|
||||
assert local_pad == gathered_pad
|
||||
|
||||
|
||||
def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch):
|
||||
"""A decode row whose top-k shard holds no local candidates (all -1) has
|
||||
undefined kernel out/lse; it must come back as (0, -inf), the identity of
|
||||
the cross-rank LSE merge, or a NaN would survive the merge even at zero
|
||||
weight (0 * NaN = NaN)."""
|
||||
num_tokens, num_heads, head_dim = 3, 2, 3
|
||||
q = torch.empty(num_tokens, num_heads, head_dim, device=DEVICE_TYPE)
|
||||
local_indices = torch.tensor(
|
||||
[[0, 1, -1, -1], [-1, -1, -1, -1], [2, -1, 3, -1]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"triton_filter_and_convert_dcp_index",
|
||||
lambda *args, **kwargs: local_indices,
|
||||
)
|
||||
|
||||
def run_kernel(**kwargs):
|
||||
out = torch.full(
|
||||
(1, num_tokens, num_heads, 1), float("nan"), device=DEVICE_TYPE
|
||||
)
|
||||
lse = torch.full((1, num_heads, num_tokens), float("nan"), device=DEVICE_TYPE)
|
||||
for token_id in (0, 2): # rows with local candidates get real values
|
||||
out[0, token_id] = float(token_id + 1)
|
||||
lse[0, :, token_id] = float(token_id + 1)
|
||||
return out, lse
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata(
|
||||
scheduler_metadata=object(), # type: ignore[arg-type]
|
||||
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
),
|
||||
req_id_per_token=torch.empty(num_tokens, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_size=64,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
impl = SimpleNamespace(
|
||||
dcp_world_size=2,
|
||||
dcp_rank=0,
|
||||
need_to_return_lse_for_decode=True,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
|
||||
out, lse = FlashMLASparseImpl._forward_fp8_kv_mixed_batch(
|
||||
impl, q, torch.empty(0, device=DEVICE_TYPE), local_indices, metadata
|
||||
)
|
||||
|
||||
assert torch.equal(out[1], torch.zeros_like(out[1]))
|
||||
assert torch.isneginf(lse[1]).all()
|
||||
for token_id in (0, 2):
|
||||
assert torch.equal(out[token_id], torch.full_like(out[token_id], token_id + 1))
|
||||
assert torch.equal(lse[token_id], torch.full_like(lse[token_id], token_id + 1))
|
||||
assert out.is_contiguous()
|
||||
assert not out.isnan().any()
|
||||
assert not lse.isnan().any()
|
||||
|
||||
@@ -343,4 +343,16 @@ class FlashMLAImpl(MLACommonImpl[FlashMLAMetadata]):
|
||||
|
||||
o = reshape_attn_output_for_spec_decode(o)
|
||||
|
||||
if self.need_to_return_lse_for_decode:
|
||||
# FlashMLA returns LSE as [batch, heads, seq_len]; the DCP reducer
|
||||
# consumes [tokens, heads]. Flattening matters under spec-decode,
|
||||
# where seq_len > 1. Only DCP consumes lse, so skip the copy
|
||||
# otherwise.
|
||||
num_decodes, q_num_heads, seq_len = lse.shape
|
||||
lse = (
|
||||
lse.permute(0, 2, 1)
|
||||
.reshape(num_decodes * seq_len, q_num_heads)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
return o, lse
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_convert_req_index_to_global_index,
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
reshape_attn_output_for_spec_decode,
|
||||
@@ -254,9 +255,12 @@ class FlashMLASparseMetadataBuilder(
|
||||
)
|
||||
else:
|
||||
threshold = {16: 128, 32: 128, 64: 256, 128: 256}.get(num_q_heads, 256)
|
||||
# Varlen decodes are safe under DCP: causality comes from the
|
||||
# indexer's top-k indices, not from the kernel metadata.
|
||||
self._init_reorder_batch_threshold(
|
||||
threshold,
|
||||
supports_spec_as_decode=True,
|
||||
supports_dcp_with_varlen=(parallel_config.cp_kv_cache_interleave_size == 1),
|
||||
)
|
||||
|
||||
sm_count = num_compute_units(device.index)
|
||||
@@ -313,6 +317,42 @@ class FlashMLASparseMetadataBuilder(
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
|
||||
|
||||
if parallel_config.decode_context_parallel_size > 1:
|
||||
if parallel_config.dcp_comm_backend != "ag_rs":
|
||||
raise NotImplementedError(
|
||||
"DCP for FlashMLA sparse is only validated with the "
|
||||
"default 'ag_rs' DCP comm backend; got "
|
||||
f"'{parallel_config.dcp_comm_backend}'"
|
||||
)
|
||||
if not self.fp8_use_mixed_batch:
|
||||
raise NotImplementedError(
|
||||
"DCP for FlashMLA sparse is only supported on the "
|
||||
"mixed-batch fp8 path (num_heads < "
|
||||
f"{MIN_HEADS_FOR_BF16_PREFILL}); the separate "
|
||||
"prefill/decode path returns the LSE for decode tokens "
|
||||
"only, while the DCP merge needs it for every token"
|
||||
)
|
||||
# Head padding (and the tile-scheduler metadata sized from it) is
|
||||
# computed from the local head count, but the kernel runs on the
|
||||
# DCP-gathered heads.
|
||||
gathered_num_heads = (
|
||||
self.num_heads * parallel_config.decode_context_parallel_size
|
||||
)
|
||||
gathered_padded_heads = FlashMLASparseImpl._compute_fp8_decode_padded_heads(
|
||||
gathered_num_heads
|
||||
)
|
||||
if self.fp8_decode_padded_heads != gathered_padded_heads:
|
||||
raise NotImplementedError(
|
||||
"DCP for FlashMLA sparse requires the local and "
|
||||
"DCP-gathered head counts to pad to the same fp8 decode "
|
||||
f"kernel envelope; got {self.num_heads} local heads "
|
||||
f"(pad to {self.fp8_decode_padded_heads}) vs "
|
||||
f"{gathered_num_heads} gathered heads (pad to "
|
||||
f"{gathered_padded_heads})"
|
||||
)
|
||||
|
||||
def _build_fp8_mixed_decode_prefill(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
@@ -496,10 +536,9 @@ class FlashMLASparseMetadataBuilder(
|
||||
) -> FlashMLASparseMetadata:
|
||||
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
|
||||
|
||||
fp8_use_mixed_batch = self.num_heads < MIN_HEADS_FOR_BF16_PREFILL
|
||||
metadata.fp8_use_mixed_batch = fp8_use_mixed_batch
|
||||
metadata.fp8_use_mixed_batch = self.fp8_use_mixed_batch
|
||||
if self.use_fp8_kv_cache:
|
||||
if fp8_use_mixed_batch:
|
||||
if self.fp8_use_mixed_batch:
|
||||
metadata.fp8_extra_metadata = self._build_fp8_mixed_decode_prefill(
|
||||
common_attn_metadata
|
||||
)
|
||||
@@ -512,6 +551,8 @@ class FlashMLASparseMetadataBuilder(
|
||||
|
||||
|
||||
class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
can_return_lse_for_decode: bool = True
|
||||
|
||||
@staticmethod
|
||||
def _compute_fp8_decode_padded_heads(num_heads: int) -> int:
|
||||
# FP8 decode kernel only supports h_q = 64 or 128
|
||||
@@ -566,6 +607,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
"fp8_ds_mla kv-cache dtype"
|
||||
)
|
||||
|
||||
if self.need_to_return_lse_for_decode and not is_quantized_kv_cache(
|
||||
kv_cache_dtype
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DCP for FlashMLA sparse requires an fp8_ds_mla kv-cache; "
|
||||
"the bf16 sparse path is not supported under DCP."
|
||||
)
|
||||
|
||||
if kv_cache_dtype == "fp8_ds_mla":
|
||||
# Reserve workspace during initialization
|
||||
assert vllm_config is not None and vllm_config.model_config is not None
|
||||
@@ -590,7 +639,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
attn_metadata: FlashMLASparseMetadata,
|
||||
) -> torch.Tensor:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill). req_id_per_token covers the whole batch; slice it
|
||||
# to the MQA tokens (q may exclude prefill tokens routed to dense MHA).
|
||||
@@ -712,7 +761,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice]
|
||||
chunk_topk_length = topk_length[chunk.tokens_slice]
|
||||
|
||||
attn_out[chunk.tokens_slice] = self._bf16_flash_mla_kernel(
|
||||
attn_out[chunk.tokens_slice], _ = self._bf16_flash_mla_kernel(
|
||||
chunk_q,
|
||||
chunk_workspace,
|
||||
chunk_topk_indices_workspace,
|
||||
@@ -727,22 +776,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
attn_metadata: FlashMLASparseMetadata,
|
||||
) -> torch.Tensor:
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Mixed batch FP8 forward path that treats all tokens as one batch.
|
||||
|
||||
This is equivalent to main branch's approach and avoids the BF16
|
||||
prefill kernel which has head padding overhead when num_heads is small.
|
||||
Used when use_mixed_batch is True.
|
||||
|
||||
The lse is only returned when DCP needs it, otherwise None.
|
||||
"""
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill).
|
||||
topk_indices = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
)
|
||||
if self.dcp_world_size > 1:
|
||||
# The indexer emits global token ids; keep this rank's shard and
|
||||
# convert to local slots. compact_valid_to_front=False keeps the
|
||||
# scattered -1s, which the fp8 kernel masks natively and the
|
||||
# empty-row neutralization below relies on. req_id is sliced to
|
||||
# topk_indices rows (the converter grids from req_id).
|
||||
topk_indices = triton_filter_and_convert_dcp_index(
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
dcp_size=self.dcp_world_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
compact_valid_to_front=False,
|
||||
)
|
||||
else:
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill).
|
||||
topk_indices = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
)
|
||||
|
||||
assert attn_metadata.fp8_extra_metadata is not None
|
||||
assert isinstance(
|
||||
@@ -750,15 +819,29 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
)
|
||||
fp8_metadata = attn_metadata.fp8_extra_metadata
|
||||
|
||||
_attn_out, _ = self._fp8_flash_mla_kernel(
|
||||
_attn_out, _lse = self._fp8_flash_mla_kernel(
|
||||
q=q.unsqueeze(0), # unsqueeze to add batch_dim: (T, H, D) -> (1, T, H, D)
|
||||
kv_c_and_k_pe_cache=kv_c_and_k_pe_cache,
|
||||
topk_indices=topk_indices.unsqueeze(0), # (T, topk) -> (1, T, topk)
|
||||
kernel_metadata=fp8_metadata,
|
||||
)
|
||||
|
||||
# Output is (1, T, H, D_v), squeeze back to (T, H, D_v)
|
||||
return _attn_out.squeeze(0)
|
||||
out = _attn_out.squeeze(0)
|
||||
|
||||
if not self.need_to_return_lse_for_decode:
|
||||
return out, None
|
||||
|
||||
# Kernel LSE is (1, H, T); the DCP merge consumes (T, H).
|
||||
lse = _lse.squeeze(0).transpose(0, 1)
|
||||
# Rows where this rank owns none of the selected tokens (all indices
|
||||
# -1) have undefined out/lse; (0, -inf) is the identity element of the
|
||||
# cross-rank LSE merge, so it drops this rank from those rows.
|
||||
empty_rows = (topk_indices == -1).all(dim=-1)
|
||||
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
|
||||
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
|
||||
# The head-padding slice above can leave `out` non-contiguous, and the
|
||||
# merge feeds it to reduce_scatter.
|
||||
return out.contiguous(), lse
|
||||
|
||||
def _fp8_flash_mla_kernel(
|
||||
self,
|
||||
@@ -793,9 +876,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
softmax_scale=self.softmax_scale,
|
||||
)
|
||||
|
||||
# Slice output back to actual head count if we padded
|
||||
# Slice output and lse back to actual head count if we padded
|
||||
if actual_num_heads < padded_num_heads:
|
||||
out = out[:, :, :actual_num_heads, :]
|
||||
lse = lse[:, :actual_num_heads, :]
|
||||
|
||||
return out, lse
|
||||
|
||||
@@ -805,35 +889,42 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = q.shape[0]
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
|
||||
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
||||
)
|
||||
|
||||
# NOTE(Chen): kernel requires num_local_head to be a multiple of
|
||||
# 64 on hopper and 128 on blackwell
|
||||
if self.num_heads % self.prefill_padding != 0:
|
||||
assert self.prefill_padding % self.num_heads == 0
|
||||
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
|
||||
# self.num_heads: under DCP the heads are all-gathered before this.
|
||||
actual_num_heads = q.shape[1]
|
||||
padded_num_heads = (
|
||||
(actual_num_heads + self.prefill_padding - 1)
|
||||
// self.prefill_padding
|
||||
* self.prefill_padding
|
||||
)
|
||||
if actual_num_heads < padded_num_heads:
|
||||
logger.warning_once(
|
||||
f"Padding num_heads from {self.num_heads} to "
|
||||
f"{self.prefill_padding} for BF16 sparse prefill kernel"
|
||||
f"Padding num_heads from {actual_num_heads} to "
|
||||
f"{padded_num_heads} for BF16 sparse prefill kernel"
|
||||
)
|
||||
q_padded = q.new_empty((q.shape[0], self.prefill_padding, q.shape[2]))
|
||||
q_padded[:, : self.num_heads, :] = q
|
||||
q_padded = q.new_empty((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
q_padded[:, :actual_num_heads, :] = q
|
||||
q = q_padded
|
||||
|
||||
topk_indices = topk_indices.view(num_tokens, 1, -1)
|
||||
output = flash_mla_sparse_fwd(
|
||||
output, _, lse = flash_mla_sparse_fwd(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
topk_indices,
|
||||
self.softmax_scale,
|
||||
topk_length=topk_length,
|
||||
)[0]
|
||||
)
|
||||
|
||||
output = output[:, : self.num_heads, :]
|
||||
return output
|
||||
output = output[:, :actual_num_heads, :]
|
||||
lse = lse[:, :actual_num_heads]
|
||||
return output, lse
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
@@ -859,12 +950,16 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
|
||||
use_fp8_cache = self.kv_cache_dtype == "fp8_ds_mla"
|
||||
|
||||
lse: torch.Tensor | None = None
|
||||
|
||||
if not use_fp8_cache:
|
||||
attn_out = self._forward_bf16_kv(
|
||||
attn_out, bf16_lse = self._forward_bf16_kv(
|
||||
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
|
||||
)
|
||||
if self.need_to_return_lse_for_decode:
|
||||
lse = bf16_lse
|
||||
elif attn_metadata.fp8_use_mixed_batch:
|
||||
attn_out = self._forward_fp8_kv_mixed_batch(
|
||||
attn_out, lse = self._forward_fp8_kv_mixed_batch(
|
||||
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
|
||||
)
|
||||
else:
|
||||
@@ -872,4 +967,4 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
q, kv_c_and_k_pe_cache, topk_indices, attn_metadata
|
||||
)
|
||||
|
||||
return attn_out, None
|
||||
return attn_out, lse
|
||||
|
||||
Reference in New Issue
Block a user