[PCP][DCP] Enable PCP+DCP on sparse-MLA models (#56157)

Signed-off-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Signed-off-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Signed-off-by: PatrykSaffer <patryk.saffer@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-1.blc1.mistralcompute.ai>
Co-authored-by: patryk.saffer <patryk.saffer@blc-login-0.blc1.mistralcompute.ai>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Patryk Saffer <PatrykSaffer@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
PatrykSaffer
2026-09-12 23:24:44 +00:00
committed by GitHub
co-authored by patryk.saffer patryk.saffer Lucas Wilkinson Patryk Saffer Claude Fable 5.1
parent 72d4d83157
commit ebe1dec2da
21 changed files with 917 additions and 138 deletions
@@ -3,6 +3,7 @@
"""Custom Sparse Attention Indexer layers."""
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm import _custom_ops as ops
@@ -130,6 +131,23 @@ def _merge_dcp_topk_global(
)
def dcp_gather_kv_rows(
local_padded: torch.Tensor,
deinterleave_idx: torch.Tensor,
gathered_buf: torch.Tensor,
out_buf: torch.Tensor,
) -> torch.Tensor:
"""All-gather this rank's KV shard and return it in global token order."""
gathered = gathered_buf[: get_dcp_group().world_size * local_padded.shape[0]]
dist.all_gather_into_tensor(
gathered, local_padded.contiguous(), group=get_dcp_group().device_group
)
return torch.index_select(
gathered, 0, deinterleave_idx, out=out_buf[: deinterleave_idx.shape[0]]
)
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
@@ -346,11 +364,19 @@ def sparse_attn_indexer(
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
current_workspace_manager().get_simultaneous(
profile_specs: list[tuple[tuple[int, ...], torch.dtype]] = [
values_spec,
scales_spec,
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
]
if use_pcp and dcp_world_size > 1:
# The PCP+DCP path takes an all-gather destination and a
# de-interleaved result.
gather_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
profile_specs.extend(gather_spec * 2)
current_workspace_manager().get_simultaneous(*profile_specs)
# Dummy allocation to simulate for peak logits tensor memory during inference.
# FP8 elements so elements == bytes
@@ -462,9 +488,27 @@ def sparse_attn_indexer(
values_spec, scales_spec = _gather_workspace_shapes(
total_seq_lens, head_dim, fp8_dtype, use_fp4_cache
)
k_quant_full, k_scale_full = workspace_manager.get_simultaneous(
# PCP + DCP needs two more pairs: the rank-major all-gather destination
# and the de-interleaved result.
pcp_chunks = [
(c, c.pcp_deinterleave_idx)
for c in prefill_metadata.chunks
if c.pcp_deinterleave_idx is not None
]
gather_specs: list[tuple[tuple[int, int], torch.dtype]] = []
if pcp_chunks:
gathered_rows = max(
dcp_world_size * c.local_total_seq_lens for c, _ in pcp_chunks
)
deinterleaved_rows = max(idx.shape[0] for _, idx in pcp_chunks)
for rows in (gathered_rows, deinterleaved_rows):
gather_specs.extend(
_gather_workspace_shapes(rows, head_dim, fp8_dtype, use_fp4_cache)
)
k_quant_full, k_scale_full, *gather_bufs = workspace_manager.get_simultaneous(
values_spec,
scales_spec,
*gather_specs,
)
for chunk in prefill_metadata.chunks:
cu_seqlen_ks = chunk.cu_seqlen_ks
@@ -481,6 +525,23 @@ def sparse_attn_indexer(
chunk.local_cu_seq_lens,
)
# PCP + DCP KV all-gather.
deinterleave_idx = chunk.pcp_deinterleave_idx
if deinterleave_idx is not None:
# local_total_seq_lens is the PCP-padded extent here, so every
# rank contributes an identically shaped shard.
gathered_values, gathered_scales, out_values, out_scales = gather_bufs
shard = k_quant[: chunk.local_total_seq_lens]
k_quant = dcp_gather_kv_rows(
shard, deinterleave_idx, gathered_values, out_values
)
k_scale = dcp_gather_kv_rows(
k_scale[: chunk.local_total_seq_lens],
deinterleave_idx,
gathered_scales,
out_scales,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
q_scale_slice = (
q_scale[chunk.token_start : chunk.token_end]
@@ -491,7 +552,7 @@ def sparse_attn_indexer(
chunk.token_start : chunk.token_end, :topk_tokens
]
if chunk.local_total_seq_lens == 0:
if chunk.local_total_seq_lens == 0 or q_slice.shape[0] == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
else:
@@ -561,15 +622,18 @@ def sparse_attn_indexer(
topk_tokens,
)
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if deinterleave_idx is None:
# Under the PCP path the top-k already ran over the whole
# context.
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
@@ -910,6 +974,10 @@ class SparseAttnIndexer(CustomOp):
if current_platform.is_cuda() or current_platform.is_xpu():
return self.forward_cuda(hidden_states, q_quant, k, weights)
elif current_platform.is_rocm():
if self.use_pcp and self.dcp_world_size > 1:
raise NotImplementedError(
"The ROCm sparse-indexer path does not support PCP+DCP."
)
return self.forward_hip(hidden_states, q_quant, k, weights)
elif current_platform.is_cpu():
return self.forward_cpu(hidden_states, q_quant, k, weights)