[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -8,7 +8,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -49,6 +49,23 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
|
||||
def _get_comm_group(self, is_sequence_parallel: bool) -> Any:
|
||||
if is_sequence_parallel:
|
||||
return get_ep_group()
|
||||
if self.dp_world_size > 1:
|
||||
return get_dp_group()
|
||||
return get_pcp_group()
|
||||
|
||||
def _get_sizes(self, num_local_tokens: int, comm_group: Any) -> list[int]:
|
||||
if self.dp_world_size == 1:
|
||||
return [num_local_tokens] * comm_group.world_size
|
||||
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
return sizes
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -62,11 +79,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
|
||||
tensors_to_gather = [hidden_states, router_logits]
|
||||
@@ -97,11 +111,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
|
||||
tensors_to_gather = [hidden_states, topk_weights, topk_ids]
|
||||
@@ -129,12 +140,11 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Reduce-scatter hidden_states across all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(
|
||||
hidden_states.shape[0] // dist_group.world_size,
|
||||
dist_group,
|
||||
)
|
||||
hidden_states = dist_group.reduce_scatterv(hidden_states, dim=0, sizes=sizes)
|
||||
return hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user