[PCP][DCP] Enable PCP+DCP on sparse-MLA models (#56157)

Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
PatrykSaffer
2026-09-12 23:24:44 +00:00
committed by GitHub
co-authored by patryk.saffer patryk.saffer Lucas Wilkinson Patryk Saffer Claude Fable 5.1
parent 72d4d83157
commit ebe1dec2da
21 changed files with 917 additions and 138 deletions
@@ -420,6 +420,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
"""
supports_dense_mha_prefill: ClassVar[bool] = True
# Under PCP+DCP only the decode rows carry an LSE; the base forward
# merges a full-batch LSE, so subclasses opt in with their own forward.
supports_pcp_dcp: ClassVar[bool] = False
def __init__(
self,
@@ -660,6 +663,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
_COMPUTE_PREFILL_METADATA_KERNEL.register_warmup()
if self.use_pcp and self.impl.dcp_world_size > 1 and not self.supports_pcp_dcp:
raise NotImplementedError(
f"{type(self).__name__} does not support PCP+DCP."
)
self.dcp_manager: MLADCPManager | None = None
if self.impl.dcp_world_size > 1:
query_dtype = (
@@ -190,7 +190,8 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
pin_memory=PIN_MEMORY,
)
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.pcp_world_size = parallel_config.prefill_context_parallel_size
self.use_pcp = self.pcp_world_size > 1
try:
self.dcp_world_size = get_dcp_group().world_size
except AssertionError:
@@ -409,8 +410,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
q_data_type=self.model_config.dtype,
output_dtype=self.model_config.dtype,
prefill_backend=self._prefill_backend,
use_dense_mha=_use_dense_mha_prefill(
self.vllm_config, prefill_max_seq_len, self.topk_tokens
use_dense_mha=(
_use_dense_mha_prefill(
self.vllm_config, prefill_max_seq_len, self.topk_tokens
)
# Dense MHA cannot be used under PCP+DCP.
and not (self.use_pcp and self.dcp_world_size > 1)
),
topk_mask_workspace=self.topk_mask_workspace,
host_staging_plan=staging_plan,
@@ -3,6 +3,7 @@
"""Custom Sparse Attention Indexer layers."""
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm import _custom_ops as ops
@@ -130,6 +131,23 @@ def _merge_dcp_topk_global(
)
def dcp_gather_kv_rows(
local_padded: torch.Tensor,
deinterleave_idx: torch.Tensor,
gathered_buf: torch.Tensor,
out_buf: torch.Tensor,
) -> torch.Tensor:
"""All-gather this rank's KV shard and return it in global token order."""
gathered = gathered_buf[: get_dcp_group().world_size * local_padded.shape[0]]
dist.all_gather_into_tensor(
gathered, local_padded.contiguous(), group=get_dcp_group().device_group
)
return torch.index_select(
gathered, 0, deinterleave_idx, out=out_buf[: deinterleave_idx.shape[0]]
)
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
@@ -346,11 +364,19 @@ def sparse_attn_indexer(
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
current_workspace_manager().get_simultaneous(
profile_specs: list[tuple[tuple[int, ...], torch.dtype]] = [
values_spec,
scales_spec,
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
]
if use_pcp and dcp_world_size > 1:
# The PCP+DCP path takes an all-gather destination and a
# de-interleaved result.
gather_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
profile_specs.extend(gather_spec * 2)
current_workspace_manager().get_simultaneous(*profile_specs)
# Dummy allocation to simulate for peak logits tensor memory during inference.
# FP8 elements so elements == bytes
@@ -462,9 +488,27 @@ def sparse_attn_indexer(
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
# PCP + DCP needs two more pairs: the rank-major all-gather destination
# and the de-interleaved result.
pcp_chunks = [
(c, c.pcp_deinterleave_idx)
for c in prefill_metadata.chunks
if c.pcp_deinterleave_idx is not None
]
gather_specs: list[tuple[tuple[int, int], torch.dtype]] = []
if pcp_chunks:
gathered_rows = max(
dcp_world_size * c.local_total_seq_lens for c, _ in pcp_chunks
)
deinterleaved_rows = max(idx.shape[0] for _, idx in pcp_chunks)
for rows in (gathered_rows, deinterleaved_rows):
gather_specs.extend(
_gather_workspace_shapes(rows, head_dim, fp8_dtype, use_fp4_cache)
)
k_quant_full, k_scale_full, *gather_bufs = workspace_manager.get_simultaneous(
values_spec,
scales_spec,
*gather_specs,
)
for chunk in prefill_metadata.chunks:
cu_seqlen_ks = chunk.cu_seqlen_ks
@@ -481,6 +525,23 @@ def sparse_attn_indexer(
chunk.local_cu_seq_lens,
)
# PCP + DCP KV all-gather.
deinterleave_idx = chunk.pcp_deinterleave_idx
if deinterleave_idx is not None:
# local_total_seq_lens is the PCP-padded extent here, so every
# rank contributes an identically shaped shard.
gathered_values, gathered_scales, out_values, out_scales = gather_bufs
shard = k_quant[: chunk.local_total_seq_lens]
k_quant = dcp_gather_kv_rows(
shard, deinterleave_idx, gathered_values, out_values
)
k_scale = dcp_gather_kv_rows(
k_scale[: chunk.local_total_seq_lens],
deinterleave_idx,
gathered_scales,
out_scales,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
@@ -491,7 +552,7 @@ def sparse_attn_indexer(
chunk.token_start : chunk.token_end, :topk_tokens
]
if chunk.local_total_seq_lens == 0:
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
else:
@@ -561,15 +622,18 @@ def sparse_attn_indexer(
topk_tokens,
)
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if deinterleave_idx is None:
# Under the PCP path the top-k already ran over the whole
# context.
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
@@ -910,6 +974,10 @@ class SparseAttnIndexer(CustomOp):
if current_platform.is_cuda() or current_platform.is_xpu():
return self.forward_cuda(hidden_states, q_quant, k, weights)
elif current_platform.is_rocm():
if self.use_pcp and self.dcp_world_size > 1:
raise NotImplementedError(
"The ROCm sparse-indexer path does not support PCP+DCP."
)
return self.forward_hip(hidden_states, q_quant, k, weights)
elif current_platform.is_cpu():
return self.forward_cpu(hidden_states, q_quant, k, weights)
@@ -919,6 +919,16 @@ class SparseAttnIndexerKpool(CustomOp):
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM to be installed."
)
_cfg = get_current_vllm_config_or_none()
_parallel = _cfg.parallel_config if _cfg is not None else None
if (
_parallel is not None
and _parallel.prefill_context_parallel_size > 1
and _parallel.decode_context_parallel_size > 1
):
raise NotImplementedError(
"SparseAttnIndexerKpool does not support PCP+DCP."
)
def forward_native(
self,