[Fast Start] Support PP (#55477)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Xun Sun <UNIDY2002@outlook.com>
This commit is contained in:
siyu
2026-09-29 09:37:43 +00:00
committed by GitHub
co-authored by Isotr0py Xun Sun
parent 77e52645e9
commit af5b4857e1
5 changed files with 100 additions and 52 deletions
@@ -250,12 +250,12 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
@@ -263,29 +263,43 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
assert plan_local_ranks(dp_tp) == [
(0, 2, 0, 0),
(1, 2, 0, 1),
(2, 3, 0, 0),
(3, 3, 0, 1),
]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
# PP adds one daemon placement per stage while preserving TP rank order.
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
assert plan_local_ranks(pp_tp) == [
(0, 0, 0, 0),
(1, 0, 0, 1),
(2, 0, 1, 0),
(3, 0, 1, 1),
]
def test_daemon_rejects_unmappable_parallelism():
@@ -302,8 +316,7 @@ def test_daemon_rejects_unmappable_parallelism():
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
with pytest.raises(ValueError, match="pipeline"):
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
@@ -335,6 +348,9 @@ def test_weight_cache_key_distinguishes_dp_ranks():
vllm_version="v",
dp_size=16,
dp_rank=3,
pp_size=2,
pp_rank=1,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]