[PCP][DCP] Enable PCP+DCP on sparse-MLA models (#56157)
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai> Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
patryk.saffer
patryk.saffer
Lucas Wilkinson
Patryk Saffer
Claude Fable 5.1
parent
72d4d83157
commit
ebe1dec2da
@@ -420,6 +420,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
"""
|
||||
|
||||
supports_dense_mha_prefill: ClassVar[bool] = True
|
||||
# Under PCP+DCP only the decode rows carry an LSE; the base forward
|
||||
# merges a full-batch LSE, so subclasses opt in with their own forward.
|
||||
supports_pcp_dcp: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -660,6 +663,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
_COMPUTE_PREFILL_METADATA_KERNEL.register_warmup()
|
||||
|
||||
if self.use_pcp and self.impl.dcp_world_size > 1 and not self.supports_pcp_dcp:
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support PCP+DCP."
|
||||
)
|
||||
self.dcp_manager: MLADCPManager | None = None
|
||||
if self.impl.dcp_world_size > 1:
|
||||
query_dtype = (
|
||||
|
||||
@@ -190,7 +190,8 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_world_size = parallel_config.prefill_context_parallel_size
|
||||
self.use_pcp = self.pcp_world_size > 1
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
except AssertionError:
|
||||
@@ -409,8 +410,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
use_dense_mha=_use_dense_mha_prefill(
|
||||
self.vllm_config, prefill_max_seq_len, self.topk_tokens
|
||||
use_dense_mha=(
|
||||
_use_dense_mha_prefill(
|
||||
self.vllm_config, prefill_max_seq_len, self.topk_tokens
|
||||
)
|
||||
# Dense MHA cannot be used under PCP+DCP.
|
||||
and not (self.use_pcp and self.dcp_world_size > 1)
|
||||
),
|
||||
topk_mask_workspace=self.topk_mask_workspace,
|
||||
host_staging_plan=staging_plan,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""Custom Sparse Attention Indexer layers."""
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
@@ -130,6 +131,23 @@ def _merge_dcp_topk_global(
|
||||
)
|
||||
|
||||
|
||||
def dcp_gather_kv_rows(
|
||||
local_padded: torch.Tensor,
|
||||
deinterleave_idx: torch.Tensor,
|
||||
gathered_buf: torch.Tensor,
|
||||
out_buf: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""All-gather this rank's KV shard and return it in global token order."""
|
||||
|
||||
gathered = gathered_buf[: get_dcp_group().world_size * local_padded.shape[0]]
|
||||
dist.all_gather_into_tensor(
|
||||
gathered, local_padded.contiguous(), group=get_dcp_group().device_group
|
||||
)
|
||||
return torch.index_select(
|
||||
gathered, 0, deinterleave_idx, out=out_buf[: deinterleave_idx.shape[0]]
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_quant_kernel(
|
||||
positions,
|
||||
@@ -346,11 +364,19 @@ def sparse_attn_indexer(
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
current_workspace_manager().get_simultaneous(
|
||||
profile_specs: list[tuple[tuple[int, ...], torch.dtype]] = [
|
||||
values_spec,
|
||||
scales_spec,
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
]
|
||||
if use_pcp and dcp_world_size > 1:
|
||||
# The PCP+DCP path takes an all-gather destination and a
|
||||
# de-interleaved result.
|
||||
gather_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
profile_specs.extend(gather_spec * 2)
|
||||
current_workspace_manager().get_simultaneous(*profile_specs)
|
||||
|
||||
# Dummy allocation to simulate for peak logits tensor memory during inference.
|
||||
# FP8 elements so elements == bytes
|
||||
@@ -462,9 +488,27 @@ def sparse_attn_indexer(
|
||||
values_spec, scales_spec = _gather_workspace_shapes(
|
||||
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
|
||||
)
|
||||
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
|
||||
# PCP + DCP needs two more pairs: the rank-major all-gather destination
|
||||
# and the de-interleaved result.
|
||||
pcp_chunks = [
|
||||
(c, c.pcp_deinterleave_idx)
|
||||
for c in prefill_metadata.chunks
|
||||
if c.pcp_deinterleave_idx is not None
|
||||
]
|
||||
gather_specs: list[tuple[tuple[int, int], torch.dtype]] = []
|
||||
if pcp_chunks:
|
||||
gathered_rows = max(
|
||||
dcp_world_size * c.local_total_seq_lens for c, _ in pcp_chunks
|
||||
)
|
||||
deinterleaved_rows = max(idx.shape[0] for _, idx in pcp_chunks)
|
||||
for rows in (gathered_rows, deinterleaved_rows):
|
||||
gather_specs.extend(
|
||||
_gather_workspace_shapes(rows, head_dim, fp8_dtype, use_fp4_cache)
|
||||
)
|
||||
k_quant_full, k_scale_full, *gather_bufs = workspace_manager.get_simultaneous(
|
||||
values_spec,
|
||||
scales_spec,
|
||||
*gather_specs,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
cu_seqlen_ks = chunk.cu_seqlen_ks
|
||||
@@ -481,6 +525,23 @@ def sparse_attn_indexer(
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
# PCP + DCP KV all-gather.
|
||||
deinterleave_idx = chunk.pcp_deinterleave_idx
|
||||
if deinterleave_idx is not None:
|
||||
# local_total_seq_lens is the PCP-padded extent here, so every
|
||||
# rank contributes an identically shaped shard.
|
||||
gathered_values, gathered_scales, out_values, out_scales = gather_bufs
|
||||
shard = k_quant[: chunk.local_total_seq_lens]
|
||||
k_quant = dcp_gather_kv_rows(
|
||||
shard, deinterleave_idx, gathered_values, out_values
|
||||
)
|
||||
k_scale = dcp_gather_kv_rows(
|
||||
k_scale[: chunk.local_total_seq_lens],
|
||||
deinterleave_idx,
|
||||
gathered_scales,
|
||||
out_scales,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
q_scale_slice = (
|
||||
q_scale[chunk.token_start : chunk.token_end]
|
||||
@@ -491,7 +552,7 @@ def sparse_attn_indexer(
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
topk_indices.fill_(-1)
|
||||
else:
|
||||
@@ -561,15 +622,18 @@ def sparse_attn_indexer(
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
)
|
||||
if deinterleave_idx is None:
|
||||
# Under the PCP path the top-k already ran over the whole
|
||||
# context.
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
@@ -910,6 +974,10 @@ class SparseAttnIndexer(CustomOp):
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
return self.forward_cuda(hidden_states, q_quant, k, weights)
|
||||
elif current_platform.is_rocm():
|
||||
if self.use_pcp and self.dcp_world_size > 1:
|
||||
raise NotImplementedError(
|
||||
"The ROCm sparse-indexer path does not support PCP+DCP."
|
||||
)
|
||||
return self.forward_hip(hidden_states, q_quant, k, weights)
|
||||
elif current_platform.is_cpu():
|
||||
return self.forward_cpu(hidden_states, q_quant, k, weights)
|
||||
|
||||
@@ -919,6 +919,16 @@ class SparseAttnIndexerKpool(CustomOp):
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM to be installed."
|
||||
)
|
||||
_cfg = get_current_vllm_config_or_none()
|
||||
_parallel = _cfg.parallel_config if _cfg is not None else None
|
||||
if (
|
||||
_parallel is not None
|
||||
and _parallel.prefill_context_parallel_size > 1
|
||||
and _parallel.decode_context_parallel_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexerKpool does not support PCP+DCP."
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user