[Core] Add MRV2 virtual-batch PCP for MLA (#46570)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-19 02:53:15 +00:00
committed by GitHub
co-authored by Codex OpenAI Codex Matthew Bonanni
parent 9243e0124e
commit b6ff8a2f50
39 changed files with 1204 additions and 196 deletions
@@ -211,6 +211,7 @@ from vllm.config import (
from vllm.config.cache import CacheDType
from vllm.distributed.parallel_state import (
get_dcp_group,
get_tp_group,
is_global_first_rank,
)
from vllm.forward_context import ForwardContext, get_forward_context
@@ -225,6 +226,10 @@ from vllm.model_executor.layers.attention.attention import (
from vllm.model_executor.layers.attention.kv_transfer_utils import (
maybe_transfer_kv_layer,
)
from vllm.model_executor.layers.attention.pcp import (
finalize_mla_pcp_decode,
maybe_gather_mla_latent_cache_inputs,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -269,7 +274,7 @@ from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
split_decodes_and_prefills,
)
from vllm.v1.attention.ops.common import cp_lse_ag_out_rs
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
from vllm.v1.attention.selector import get_attn_backend
@@ -499,6 +504,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.use_direct_call = not current_platform.opaque_attention_op()
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
@@ -611,11 +618,23 @@ class MLAAttention(nn.Module, AttentionLayerBase):
assert isinstance(slot_mapping, dict), (
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
)
layer_slot_mapping = slot_mapping.get(self.layer_name)
kv_for_cache, kpe_for_cache, layer_slot_mapping = (
maybe_gather_mla_latent_cache_inputs(
kv_c_normed,
k_pe,
layer_slot_mapping,
attn_metadata.num_decode_tokens
if attn_metadata is not None
else None,
self.use_pcp,
)
)
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_c_normed,
k_pe,
kv_for_cache,
kpe_for_cache,
self_kv_cache,
slot_mapping.get(self.layer_name),
layer_slot_mapping,
self.kv_cache_dtype,
self._k_scale,
)
@@ -710,6 +729,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
num_actual_toks = attn_metadata.num_actual_tokens
if self.use_pcp and self.impl.dcp_world_size > 1 and quant_key is not None:
raise NotImplementedError(
"MRV2 MLA PCP+DCP does not support fused output quantization yet."
)
# Inputs and outputs may be padded for CUDA graphs
output_padded = output
@@ -833,11 +856,15 @@ class MLAAttention(nn.Module, AttentionLayerBase):
else:
mqa_q = (mqa_ql_nope, mqa_q_pe)
if self.impl.dcp_world_size > 1:
if isinstance(mqa_q, tuple):
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
mqa_q = torch.cat(mqa_q, dim=-1)
# mqa_q do allgather in head dim.
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
if self.use_pcp:
if self.impl.dcp_world_size > self.impl.pcp_world_size:
if isinstance(mqa_q, tuple):
mqa_q = torch.cat(mqa_q, dim=-1)
mqa_q = get_tp_group().all_gather(mqa_q, dim=1)
else:
if isinstance(mqa_q, tuple):
mqa_q = torch.cat(mqa_q, dim=-1)
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
# call decode attn
if not self.impl.is_sparse:
@@ -846,6 +873,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# correct dcp attn_out with lse.
if self.impl.dcp_world_size > 1:
assert lse is not None
if self.dcp_a2a:
attn_out = dcp_a2a_lse_reduce(
attn_out,
@@ -853,6 +881,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
elif self.use_pcp:
attn_out = cp_lse_ag_out_ar(
attn_out,
lse,
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
else:
attn_out = cp_lse_ag_out_rs(
attn_out,
@@ -860,6 +895,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
get_dcp_group(),
is_lse_base_on_e=self.impl.lse_base_on_e,
)
if self.use_pcp:
attn_out = finalize_mla_pcp_decode(attn_out, self.num_heads)
# v_up projection
self._v_up_proj(attn_out, out=mqa_output_slice)
@@ -903,6 +940,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
raise ValueError(f"Unsupported quant_key: {quant_key}")
return quant_output
if self.use_pcp and output_padded.shape[0] > num_actual_toks:
output_padded[num_actual_toks:].zero_()
return output_padded
def process_weights_after_loading(self, act_dtype: torch.dtype):
@@ -1082,8 +1121,17 @@ def unified_mla_kv_cache_update(
the data dependency between them to ensure torch.compile preserves ordering.
"""
layer_name = _resolve_layer_name(layer_name)
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
layer_name
)
if layer_slot_mapping is not None:
kv_c_normed, k_pe, layer_slot_mapping = maybe_gather_mla_latent_cache_inputs(
kv_c_normed,
k_pe,
layer_slot_mapping,
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
attn_layer.use_pcp,
)
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
kv_c_normed,
k_pe,
@@ -1723,6 +1771,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.compilation_config = vllm_config.compilation_config
self.vllm_config = vllm_config
self.device = device
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
@@ -1735,11 +1784,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
try:
self.dcp_world_size = get_dcp_group().world_size
self.dcp_rank = get_dcp_group().rank_in_group
except AssertionError:
# DCP might not be initialized in testing
self.dcp_world_size = 1
self.dcp_rank = 0
self.dcp_local_block_size = parallel_config.cp_kv_cache_interleave_size
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
@@ -1857,6 +1904,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=(self.query_len_support != QueryLenSupport.VARLEN),
treat_short_extends_as_decodes=not self.use_pcp,
)
)
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tp_group,
)
def _gather_prefill_cache_inputs(
tensors: tuple[torch.Tensor, ...],
slot_mapping: torch.Tensor,
num_decode_tokens: int,
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
"""Keep replicated decode writes local and gather partitioned prefills."""
local_num_tokens = tensors[0].shape[0]
assert all(tensor.shape[0] == local_num_tokens for tensor in tensors)
assert 0 <= num_decode_tokens <= local_num_tokens
if num_decode_tokens == local_num_tokens:
return tensors, slot_mapping[:num_decode_tokens]
pcp_group = get_pcp_group()
gathered_prefills = tuple(
pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0)
for tensor in tensors
)
pcp_size = pcp_group.world_size
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
if num_decode_tokens == 0:
return gathered_prefills, gathered_slot_mapping
cache_inputs = tuple(
torch.cat((tensor[:num_decode_tokens], gathered_prefill), dim=0)
for tensor, gathered_prefill in zip(tensors, gathered_prefills)
)
rank_slot_mappings = gathered_slot_mapping.view(pcp_size, local_num_tokens)
cache_slot_mapping = torch.cat(
(
rank_slot_mappings[0, :num_decode_tokens],
rank_slot_mappings[:, num_decode_tokens:].flatten(),
)
)
return cache_inputs, cache_slot_mapping
def maybe_gather_mla_latent_cache_inputs(
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
slot_mapping: torch.Tensor | None,
num_decode_tokens: int | None,
use_pcp: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
if not use_pcp or num_decode_tokens is None:
return kv_c_normed, k_pe, slot_mapping
assert slot_mapping is not None
num_tokens = kv_c_normed.shape[0]
k_pe_flat = k_pe.reshape(num_tokens, -1)
(cache_kv_c, cache_k_pe_flat), cache_slot_mapping = _gather_prefill_cache_inputs(
(kv_c_normed, k_pe_flat),
slot_mapping,
num_decode_tokens,
)
cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:])
return cache_kv_c, cache_k_pe, cache_slot_mapping
def maybe_gather_indexer_k(
k: torch.Tensor,
slot_mapping: torch.Tensor,
num_decode_tokens: int,
use_pcp: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if not use_pcp:
return k, slot_mapping
(cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs(
(k,), slot_mapping, num_decode_tokens
)
return cache_k, cache_slot_mapping
def finalize_mla_pcp_decode(
output: torch.Tensor,
num_heads: int,
) -> torch.Tensor:
if output.shape[1] < num_heads:
output = get_pcp_group().all_gather(output, dim=1)
elif output.shape[1] > num_heads:
head_start = get_tp_group().rank_in_group * num_heads
output = output[:, head_start : head_start + num_heads]
return output
@@ -59,13 +59,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
device=device,
)
parallel_config = vllm_config.parallel_config
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
try:
self.dcp_world_size = get_dcp_group().world_size
self.dcp_rank = get_dcp_group().rank_in_group
except AssertionError:
# DCP might not be initialized in testing
self.dcp_world_size = 1
self.dcp_rank = 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
self.dcp_local_block_size = self.cp_kv_cache_interleave_size
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
@@ -174,6 +173,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold or 1,
treat_short_extends_as_decodes=not self.use_pcp,
require_uniform=self.require_uniform_decodes,
)
(
@@ -1050,7 +1050,9 @@ class FusedMoEParallelConfig:
@property
def use_all2all_kernels(self):
return self.use_ep and (self.dp_size > 1 or self.is_sequence_parallel)
return self.use_ep and (
self.dp_size > 1 or self.pcp_size > 1 or self.is_sequence_parallel
)
@property
def use_deepep_ht_kernels(self):
@@ -752,18 +752,12 @@ class MoERunner(MoERunnerInterface):
assert len(result) == 2
hidden_states, router_logits = result
# NOTE: Similar with DP, PCP also needs dispatch and combine. For
# simplicity, AgRsAll2All was added separately for PCP here. Maybe
# we should modify All2AllManager abstraction to better support PCP.
if self.moe_config.pcp_size > 1:
hidden_states = get_pcp_group().all_gather(
hidden_states,
dim=0,
)
router_logits = get_pcp_group().all_gather(
router_logits,
dim=0,
)
if (
self.moe_config.pcp_size > 1
and not self.moe_config.moe_parallel_config.use_all2all_kernels
):
hidden_states = get_pcp_group().all_gather(hidden_states, dim=0)
router_logits = get_pcp_group().all_gather(router_logits, dim=0)
return hidden_states, router_logits
@@ -777,11 +771,11 @@ class MoERunner(MoERunnerInterface):
hidden_states, self.moe_config.is_sequence_parallel
)
if self.moe_config.pcp_size > 1:
hidden_states = get_pcp_group().reduce_scatter(
hidden_states,
dim=0,
)
if (
self.moe_config.pcp_size > 1
and not self.moe_config.moe_parallel_config.use_all2all_kernels
):
hidden_states = get_pcp_group().reduce_scatter(hidden_states, dim=0)
if self.shared_experts is not None:
assert shared_output is not None
@@ -9,10 +9,11 @@ from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dcp_group
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.attention.pcp import maybe_gather_indexer_k
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
@@ -308,6 +309,7 @@ def sparse_attn_indexer(
total_seq_lens: int,
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_pcp: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
@@ -354,6 +356,7 @@ def sparse_attn_indexer(
total_seq_lens,
topk_indices_buffer,
skip_k_cache_insert,
use_pcp,
use_fp4_cache,
)
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
@@ -373,18 +376,28 @@ def sparse_attn_indexer(
# During speculative decoding, k may be padded to the CUDA graph batch
# size while slot_mapping only covers actual tokens. Truncate k to avoid
# out-of-bounds reads in the kernel.
# Keep PCP padding so every rank contributes the same all-gather shape.
num_tokens = slot_mapping.shape[0]
if use_pcp:
num_tokens //= get_pcp_group().world_size
if k is not None:
k = k[:num_tokens]
if not skip_k_cache_insert:
assert k is not None
k, slot_mapping_for_cache = maybe_gather_indexer_k(
k,
slot_mapping,
num_decode_tokens,
use_pcp,
)
# scale_fmt can be None, but the function expects str
assert scale_fmt is not None
assert not use_fp4_cache, "Unfused FP4 Insert is not supported yet"
ops.indexer_k_quant_and_cache(
k,
kv_cache,
slot_mapping,
slot_mapping_for_cache,
quant_block_size,
scale_fmt,
)
@@ -498,7 +511,14 @@ def sparse_attn_indexer(
assert decode_metadata is not None
kv_cache = kv_cache_as_quant_view(kv_cache, head_dim, use_fp4_cache)
decode_lens = decode_metadata.decode_lens
if decode_metadata.requires_padding:
if num_decode_tokens == 0:
padded_q_quant_decode_tokens = q_quant[:1].reshape(1, 1, *q_quant.shape[1:])
padded_q_scale = (
q_scale[:1].reshape(1, 1, *q_scale.shape[1:])
if q_scale is not None
else None
)
elif decode_metadata.requires_padding:
# pad in edge case where we have short chunked prefill length <
# decode_threshold since we unstrictly split
# prefill and decode by decode_threshold
@@ -663,6 +683,7 @@ def sparse_attn_indexer_fake(
total_seq_lens: int,
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_pcp: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
@@ -725,6 +746,7 @@ class SparseAttnIndexer(CustomOp):
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
@@ -777,6 +799,7 @@ class SparseAttnIndexer(CustomOp):
self.max_total_seq_len,
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_pcp,
self.use_fp4_cache,
self.dcp_rank,
self.dcp_world_size,