[PCP] Support prefill context parallelism with data parallelism (#57075)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
QiuChunshuo
Claude Fable 5.1
Codex
Robert Shaw
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
6697a7dd30
commit
e6dc16cebd
@@ -56,11 +56,17 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self.use_ep = get_current_vllm_config().parallel_config.enable_expert_parallel
|
||||
|
||||
def _get_comm_group(self, is_sequence_parallel: bool) -> Any:
|
||||
if is_sequence_parallel:
|
||||
return get_ep_group()
|
||||
if self.dp_world_size > 1:
|
||||
if self.use_ep and get_pcp_group().world_size > 1:
|
||||
assert self.tp_group.world_size == 1, (
|
||||
"DP+PCP with TP>1 requires sequence-parallel MoE inputs"
|
||||
)
|
||||
return get_ep_group()
|
||||
return get_dp_group()
|
||||
return get_pcp_group()
|
||||
|
||||
@@ -72,6 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
assert len(sizes) == comm_group.world_size
|
||||
return sizes
|
||||
|
||||
def dispatch_router_logits(
|
||||
|
||||
@@ -6,6 +6,7 @@ from vllm.distributed.device_communicators.pynccl import defer_comm_warmup_on_ro
|
||||
from vllm.distributed.parallel_state import (
|
||||
_init_stateless_group,
|
||||
_node_count,
|
||||
get_pcp_group,
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
@@ -72,12 +73,13 @@ def create_standby_groups(
|
||||
_STANDBY_WORLD_NODE_COUNT = _node_count(_STANDBY_WORLD.tcp_store_group)
|
||||
|
||||
tp_size = get_tp_group().world_size
|
||||
pcp_size = get_pcp_group().world_size
|
||||
pp_size = get_pp_group().world_size
|
||||
|
||||
all_ranks = torch.arange(new_world_size_across_dp).reshape(
|
||||
-1, new_dp_size, pp_size, tp_size
|
||||
-1, new_dp_size, pp_size, pcp_size, tp_size
|
||||
)
|
||||
standby_dp_ranks = all_ranks.transpose(1, 3).reshape(-1, new_dp_size).unbind(0)
|
||||
standby_dp_ranks = all_ranks.transpose(1, 4).reshape(-1, new_dp_size).unbind(0)
|
||||
standby_dp_ranks = [x.tolist() for x in standby_dp_ranks]
|
||||
|
||||
# Deferred to commit so the warm-up runs while the engine is paused.
|
||||
@@ -87,7 +89,9 @@ def create_standby_groups(
|
||||
)
|
||||
|
||||
standby_ep_ranks = (
|
||||
all_ranks.transpose(1, 2).reshape(-1, new_dp_size * tp_size).unbind(0)
|
||||
all_ranks.transpose(1, 2)
|
||||
.reshape(-1, new_dp_size * pcp_size * tp_size)
|
||||
.unbind(0)
|
||||
)
|
||||
standby_ep_ranks = [x.tolist() for x in standby_ep_ranks]
|
||||
_STANDBY_EP = _init_stateless_group(
|
||||
|
||||
Reference in New Issue
Block a user