[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)

Signed-off-by: shivampr <shivampr.dev@gmail.com>
Signed-off-by: Shivam <shivamprasad91@gmail.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
shivampr
2026-06-24 19:28:17 +00:00
committed by GitHub
co-authored by Matthew Bonanni mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d511b5bae9
commit 49f2104c53
4 changed files with 359 additions and 15 deletions
@@ -800,9 +800,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
else:
mqa_q = (mqa_ql_nope, mqa_q_pe)
if self.impl.dcp_world_size > 1:
assert not fp8_attention, "DCP not support fp8 kvcache now."
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
if isinstance(mqa_q, tuple):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
@@ -1263,6 +1263,7 @@ class MLACommonPrefillMetadata:
padded_local_chunk_seq_lens: list[list[int]] | None = None
local_context_lens_allranks: list[list[int]] | None = None
padded_local_cu_seq_lens: torch.Tensor | None = None
padded_local_token_to_seq: torch.Tensor | None = None
cu_seq_lens_lst: list[list[int]] | None = None
chunk_size: int | None = None
prefill_tokens_with_context: int | None = None
@@ -1787,6 +1788,21 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
dtype=torch.int32,
)
max_padded_local_tokens_over_chunk = (
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
)
padded_local_token_to_seq_tensor_cpu = torch.zeros(
[num_chunks, max_padded_local_tokens_over_chunk],
dtype=torch.int32,
)
for i in range(num_chunks):
chunk_token_to_seq_tensor = torch.repeat_interleave(
range_idx, padded_local_chunk_seq_lens[i]
)
chunk_len = chunk_token_to_seq_tensor.shape[0]
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
chunk_token_to_seq_tensor
)
prefill_tokens_with_context = None
if num_prefills_with_context_cpu > 0:
@@ -1811,6 +1827,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
device, non_blocking=True
),
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
device, non_blocking=True
),
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
chunk_size=padded_local_max_context_chunk_across_ranks,
prefill_tokens_with_context=prefill_tokens_with_context,
@@ -2187,7 +2206,6 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
k_scale: torch.Tensor,
dcp_world_size: int,
):
assert k_scale is None, "DCP not support scaled kvcache now."
assert attn_metadata.prefill is not None
prefill_metadata = attn_metadata.prefill
assert prefill_metadata.prefill_backend is not None
@@ -2195,9 +2213,11 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None
assert prefill_metadata.chunked_context.local_context_lens_allranks is not None
assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None
assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None
assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None
assert prefill_metadata.chunked_context.chunk_size is not None
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
output = None
merge_output = None
iters = len(prefill_metadata.chunked_context.seq_tot)
@@ -2205,16 +2225,37 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
for i in range(iters):
toks = prefill_metadata.chunked_context.seq_tot[i]
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=prefill_metadata.chunked_context.padded_local_cu_seq_lens[
i
],
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
if toks == 0:
continue
padded_local_cu_seq_lens = (
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
)
if is_quantized_kv_cache(self.kv_cache_dtype) and (
self.kv_cache_dtype != "fp8_ds_mla"
):
assert k_scale is not None
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=padded_local_cu_seq_lens,
token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[
i
],
num_tokens=toks,
kv_cache_dtype=self.kv_cache_dtype,
scale=k_scale,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
else:
ops.cp_gather_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
block_table=prefill_metadata.block_table,
cu_seq_lens=padded_local_cu_seq_lens,
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
# workspace
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
# |<- use for local_gather ->|<--------- use for allgather -------->|
@@ -2252,9 +2293,22 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
toks=toks,
)
kv_b_proj_w_dtype = (
self.kv_b_proj.weight.dtype
if hasattr(self.kv_b_proj, "weight")
else self.kv_b_proj.params_dtype
)
if (
use_fp8_prefill or kv_b_proj_w_dtype != current_platform.fp8_dtype()
) and kv_b_proj_w_dtype != torch.uint8:
kv_c_normed = kv_c_normed.to(kv_b_proj_w_dtype)
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
)
if use_fp8_prefill:
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
k_pe = k_pe.to(prefill_metadata.q_data_type)
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
k = self._concat_k_nope_k_pe(k_nope, k_pe)
@@ -2346,7 +2400,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
q,
kv_c_and_k_pe_cache,
attn_metadata,
k_scale=None,
k_scale=k_scale,
dcp_world_size=self.dcp_world_size,
)
)