[Core] Add MRV2 virtual-batch PCP for MLA (#46570)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-19 02:53:15 +00:00
committed by GitHub
co-authored by Codex OpenAI Codex Matthew Bonanni
parent 9243e0124e
commit b6ff8a2f50
39 changed files with 1204 additions and 196 deletions
@@ -8,7 +8,7 @@ import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm.distributed import get_dp_group, get_ep_group
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
from vllm.distributed.utils import StatelessProcessGroup
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
@@ -49,6 +49,23 @@ class AgRsAll2AllManager(All2AllManagerBase):
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
def _get_comm_group(self, is_sequence_parallel: bool) -> Any:
if is_sequence_parallel:
return get_ep_group()
if self.dp_world_size > 1:
return get_dp_group()
return get_pcp_group()
def _get_sizes(self, num_local_tokens: int, comm_group: Any) -> list[int]:
if self.dp_world_size == 1:
return [num_local_tokens] * comm_group.world_size
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
return sizes
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
@@ -62,11 +79,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
tensors_to_gather = [hidden_states, router_logits]
@@ -97,11 +111,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
tensors_to_gather = [hidden_states, topk_weights, topk_ids]
@@ -129,12 +140,11 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Reduce-scatter hidden_states across all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
dist_group,
)
hidden_states = dist_group.reduce_scatterv(hidden_states, dim=0, sizes=sizes)
return hidden_states
@@ -176,11 +176,16 @@ class DeviceCommunicatorBase:
config = get_current_vllm_config_or_none()
if config is not None:
# initialize the all2all manager for DP or sequence-parallel EP.
parallel_config = config.parallel_config
use_ep = (
config.parallel_config.data_parallel_size > 1
or config.parallel_config.use_sequence_parallel_moe
parallel_config.data_parallel_size > 1
or parallel_config.use_sequence_parallel_moe
or (
parallel_config.enable_expert_parallel
and parallel_config.prefill_context_parallel_size > 1
)
)
all2all_backend = config.parallel_config.all2all_backend
all2all_backend = parallel_config.all2all_backend
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
self.use_all2all = self.is_ep_communicator and use_ep
@@ -36,13 +36,12 @@ def build_offloading_config(
engine_id = kv_transfer_config.engine_id
parallel_config = vllm_config.parallel_config
context_parallel_factor = (
parallel_config.decode_context_parallel_size
* parallel_config.prefill_context_parallel_size
)
groups = tuple(
OffloadingGroupConfig(
tokens_per_block=(group.kv_cache_spec.block_size * context_parallel_factor),
tokens_per_block=(
group.kv_cache_spec.block_size
* parallel_config.decode_context_parallel_size
),
layer_names=tuple(group.layer_names),
)
for group in kv_cache_config.kv_cache_groups
+14 -11
View File
@@ -1784,7 +1784,7 @@ def initialize_model_parallel(
get_world_group().device_group
)
# the layout order is: ExternalDP x DP x PP x TP
# the layout order is: ExternalDP x DP x PP x PCP x TP
# ExternalDP is the data parallel group that is not part of the model,
# every dp rank can generate independently (in verl integration).
# DP is the data parallel group that is part of the model,
@@ -1821,17 +1821,13 @@ def initialize_model_parallel(
# Build the DCP model-parallel groups.
global _DCP
assert _DCP is None, "decode context model parallel group is already initialized"
# Note(hc): In the current implementation of decode context parallel,
# dcp_size must not exceed tp_size, because the world size does not
# change by DCP, it simply reuses the GPUs of TP group, and split one
# TP group into tp_size//dcp_size DCP groups.
group_ranks = all_ranks.reshape(-1, decode_context_model_parallel_size).unbind(0)
dcp_size = decode_context_model_parallel_size or 1
dcp_ranks = local_all_ranks if enable_elastic_ep else all_ranks
if dcp_size > 1:
# DCP spans PCP first, then TP for full TP x PCP groups.
dcp_ranks = dcp_ranks.transpose(-1, -2)
group_ranks = dcp_ranks.reshape(-1, dcp_size).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
if enable_elastic_ep:
group_ranks = local_all_ranks.reshape(
-1, decode_context_model_parallel_size
).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
_DCP = init_model_parallel_group(
group_ranks,
get_world_group().local_rank,
@@ -2005,6 +2001,13 @@ def ensure_model_parallel_initialized(
f"{pcp_world_size=} vs. "
f"{prefill_context_model_parallel_size=}"
)
dcp_world_size = get_dcp_group().world_size
dcp_model_parallel_size = decode_context_model_parallel_size or 1
assert dcp_world_size == dcp_model_parallel_size, (
"decode context parallel group already initialized, but of unexpected size: "
f"{dcp_world_size=} vs. "
f"{dcp_model_parallel_size=}"
)
def prepare_communication_buffer_for_model(model: torch.nn.Module):