[Core] Add MRV2 virtual-batch PCP for MLA (#46570)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-19 02:53:15 +00:00
committed by GitHub
co-authored by Codex OpenAI Codex Matthew Bonanni
parent 9243e0124e
commit b6ff8a2f50
39 changed files with 1204 additions and 196 deletions
@@ -8,7 +8,7 @@ import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm.distributed import get_dp_group, get_ep_group
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
from vllm.distributed.utils import StatelessProcessGroup
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
@@ -49,6 +49,23 @@ class AgRsAll2AllManager(All2AllManagerBase):
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
def _get_comm_group(self, is_sequence_parallel: bool) -> Any:
if is_sequence_parallel:
return get_ep_group()
if self.dp_world_size > 1:
return get_dp_group()
return get_pcp_group()
def _get_sizes(self, num_local_tokens: int, comm_group: Any) -> list[int]:
if self.dp_world_size == 1:
return [num_local_tokens] * comm_group.world_size
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
return sizes
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
@@ -62,11 +79,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
tensors_to_gather = [hidden_states, router_logits]
@@ -97,11 +111,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
tensors_to_gather = [hidden_states, topk_weights, topk_ids]
@@ -129,12 +140,11 @@ class AgRsAll2AllManager(All2AllManagerBase):
"""
Reduce-scatter hidden_states across all dp ranks.
"""
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
dist_group,
)
hidden_states = dist_group.reduce_scatterv(hidden_states, dim=0, sizes=sizes)
return hidden_states