[Core] Add MRV2 virtual-batch PCP for MLA (#46570)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Matthew Bonanni
parent
9243e0124e
commit
b6ff8a2f50
@@ -8,7 +8,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.distributed import get_dp_group, get_ep_group, get_pcp_group
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -49,6 +49,23 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
|
||||
def _get_comm_group(self, is_sequence_parallel: bool) -> Any:
|
||||
if is_sequence_parallel:
|
||||
return get_ep_group()
|
||||
if self.dp_world_size > 1:
|
||||
return get_dp_group()
|
||||
return get_pcp_group()
|
||||
|
||||
def _get_sizes(self, num_local_tokens: int, comm_group: Any) -> list[int]:
|
||||
if self.dp_world_size == 1:
|
||||
return [num_local_tokens] * comm_group.world_size
|
||||
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
return sizes
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -62,11 +79,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
|
||||
tensors_to_gather = [hidden_states, router_logits]
|
||||
@@ -97,11 +111,8 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
|
||||
tensors_to_gather = [hidden_states, topk_weights, topk_ids]
|
||||
@@ -129,12 +140,11 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
Reduce-scatter hidden_states across all dp ranks.
|
||||
"""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
|
||||
dist_group = get_ep_group() if is_sequence_parallel else get_dp_group()
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(
|
||||
hidden_states.shape[0] // dist_group.world_size,
|
||||
dist_group,
|
||||
)
|
||||
hidden_states = dist_group.reduce_scatterv(hidden_states, dim=0, sizes=sizes)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -176,11 +176,16 @@ class DeviceCommunicatorBase:
|
||||
config = get_current_vllm_config_or_none()
|
||||
if config is not None:
|
||||
# initialize the all2all manager for DP or sequence-parallel EP.
|
||||
parallel_config = config.parallel_config
|
||||
use_ep = (
|
||||
config.parallel_config.data_parallel_size > 1
|
||||
or config.parallel_config.use_sequence_parallel_moe
|
||||
parallel_config.data_parallel_size > 1
|
||||
or parallel_config.use_sequence_parallel_moe
|
||||
or (
|
||||
parallel_config.enable_expert_parallel
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
)
|
||||
)
|
||||
all2all_backend = config.parallel_config.all2all_backend
|
||||
all2all_backend = parallel_config.all2all_backend
|
||||
|
||||
self.is_ep_communicator = unique_name.split(":")[0] == "ep"
|
||||
self.use_all2all = self.is_ep_communicator and use_ep
|
||||
|
||||
@@ -36,13 +36,12 @@ def build_offloading_config(
|
||||
engine_id = kv_transfer_config.engine_id
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
context_parallel_factor = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.prefill_context_parallel_size
|
||||
)
|
||||
groups = tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=(group.kv_cache_spec.block_size * context_parallel_factor),
|
||||
tokens_per_block=(
|
||||
group.kv_cache_spec.block_size
|
||||
* parallel_config.decode_context_parallel_size
|
||||
),
|
||||
layer_names=tuple(group.layer_names),
|
||||
)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
|
||||
@@ -1784,7 +1784,7 @@ def initialize_model_parallel(
|
||||
get_world_group().device_group
|
||||
)
|
||||
|
||||
# the layout order is: ExternalDP x DP x PP x TP
|
||||
# the layout order is: ExternalDP x DP x PP x PCP x TP
|
||||
# ExternalDP is the data parallel group that is not part of the model,
|
||||
# every dp rank can generate independently (in verl integration).
|
||||
# DP is the data parallel group that is part of the model,
|
||||
@@ -1821,17 +1821,13 @@ def initialize_model_parallel(
|
||||
# Build the DCP model-parallel groups.
|
||||
global _DCP
|
||||
assert _DCP is None, "decode context model parallel group is already initialized"
|
||||
# Note(hc): In the current implementation of decode context parallel,
|
||||
# dcp_size must not exceed tp_size, because the world size does not
|
||||
# change by DCP, it simply reuses the GPUs of TP group, and split one
|
||||
# TP group into tp_size//dcp_size DCP groups.
|
||||
group_ranks = all_ranks.reshape(-1, decode_context_model_parallel_size).unbind(0)
|
||||
dcp_size = decode_context_model_parallel_size or 1
|
||||
dcp_ranks = local_all_ranks if enable_elastic_ep else all_ranks
|
||||
if dcp_size > 1:
|
||||
# DCP spans PCP first, then TP for full TP x PCP groups.
|
||||
dcp_ranks = dcp_ranks.transpose(-1, -2)
|
||||
group_ranks = dcp_ranks.reshape(-1, dcp_size).unbind(0)
|
||||
group_ranks = [x.tolist() for x in group_ranks]
|
||||
if enable_elastic_ep:
|
||||
group_ranks = local_all_ranks.reshape(
|
||||
-1, decode_context_model_parallel_size
|
||||
).unbind(0)
|
||||
group_ranks = [x.tolist() for x in group_ranks]
|
||||
_DCP = init_model_parallel_group(
|
||||
group_ranks,
|
||||
get_world_group().local_rank,
|
||||
@@ -2005,6 +2001,13 @@ def ensure_model_parallel_initialized(
|
||||
f"{pcp_world_size=} vs. "
|
||||
f"{prefill_context_model_parallel_size=}"
|
||||
)
|
||||
dcp_world_size = get_dcp_group().world_size
|
||||
dcp_model_parallel_size = decode_context_model_parallel_size or 1
|
||||
assert dcp_world_size == dcp_model_parallel_size, (
|
||||
"decode context parallel group already initialized, but of unexpected size: "
|
||||
f"{dcp_world_size=} vs. "
|
||||
f"{dcp_model_parallel_size=}"
|
||||
)
|
||||
|
||||
|
||||
def prepare_communication_buffer_for_model(model: torch.nn.Module):
|
||||
|
||||
Reference in New Issue
Block a user