[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)
Signed-off-by: shivampr <shivampr.dev@gmail.com> Signed-off-by: Shivam <shivamprasad91@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Bonanni
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d511b5bae9
commit
49f2104c53
@@ -800,9 +800,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
else:
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert not fp8_attention, "DCP not support fp8 kvcache now."
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
if isinstance(mqa_q, tuple):
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
# mqa_q do allgather in head dim.
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
|
||||
@@ -1263,6 +1263,7 @@ class MLACommonPrefillMetadata:
|
||||
padded_local_chunk_seq_lens: list[list[int]] | None = None
|
||||
local_context_lens_allranks: list[list[int]] | None = None
|
||||
padded_local_cu_seq_lens: torch.Tensor | None = None
|
||||
padded_local_token_to_seq: torch.Tensor | None = None
|
||||
cu_seq_lens_lst: list[list[int]] | None = None
|
||||
chunk_size: int | None = None
|
||||
prefill_tokens_with_context: int | None = None
|
||||
@@ -1787,6 +1788,21 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_padded_local_tokens_over_chunk = (
|
||||
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
|
||||
)
|
||||
padded_local_token_to_seq_tensor_cpu = torch.zeros(
|
||||
[num_chunks, max_padded_local_tokens_over_chunk],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
for i in range(num_chunks):
|
||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
||||
range_idx, padded_local_chunk_seq_lens[i]
|
||||
)
|
||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
||||
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
|
||||
chunk_token_to_seq_tensor
|
||||
)
|
||||
|
||||
prefill_tokens_with_context = None
|
||||
if num_prefills_with_context_cpu > 0:
|
||||
@@ -1811,6 +1827,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk_across_ranks,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
@@ -2187,7 +2206,6 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
k_scale: torch.Tensor,
|
||||
dcp_world_size: int,
|
||||
):
|
||||
assert k_scale is None, "DCP not support scaled kvcache now."
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
@@ -2195,9 +2213,11 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.local_context_lens_allranks is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None
|
||||
assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None
|
||||
assert prefill_metadata.chunked_context.chunk_size is not None
|
||||
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
output = None
|
||||
merge_output = None
|
||||
iters = len(prefill_metadata.chunked_context.seq_tot)
|
||||
@@ -2205,16 +2225,37 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
|
||||
for i in range(iters):
|
||||
toks = prefill_metadata.chunked_context.seq_tot[i]
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=prefill_metadata.chunked_context.padded_local_cu_seq_lens[
|
||||
i
|
||||
],
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
if toks == 0:
|
||||
continue
|
||||
padded_local_cu_seq_lens = (
|
||||
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
|
||||
)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype) and (
|
||||
self.kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
assert k_scale is not None
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[
|
||||
i
|
||||
],
|
||||
num_tokens=toks,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
else:
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
# workspace
|
||||
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
|
||||
# |<- use for local_gather ->|<--------- use for allgather -------->|
|
||||
@@ -2252,9 +2293,22 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
toks=toks,
|
||||
)
|
||||
|
||||
kv_b_proj_w_dtype = (
|
||||
self.kv_b_proj.weight.dtype
|
||||
if hasattr(self.kv_b_proj, "weight")
|
||||
else self.kv_b_proj.params_dtype
|
||||
)
|
||||
if (
|
||||
use_fp8_prefill or kv_b_proj_w_dtype != current_platform.fp8_dtype()
|
||||
) and kv_b_proj_w_dtype != torch.uint8:
|
||||
kv_c_normed = kv_c_normed.to(kv_b_proj_w_dtype)
|
||||
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
|
||||
@@ -2346,7 +2400,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
k_scale=None,
|
||||
k_scale=k_scale,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user