[Fast Start] Support data parallelism in the weight cache daemon (#57386)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -251,3 +251,116 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
assert restart_outputs == baseline_outputs
|
||||
|
||||
|
||||
def _parallel(**kw):
|
||||
from types import SimpleNamespace
|
||||
|
||||
base = dict(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
nnodes=1,
|
||||
node_rank=0,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
"""Each node serves a contiguous global-rank block (DP-major), so without
|
||||
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
|
||||
by --data-parallel-start-rank."""
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
|
||||
|
||||
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
|
||||
|
||||
dp_third_node = _parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
|
||||
)
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
|
||||
|
||||
dp_tp = _parallel(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=4,
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=2,
|
||||
)
|
||||
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
|
||||
|
||||
# DP + nnodes: node-local TP replicas
|
||||
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
|
||||
dp_tp_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
|
||||
|
||||
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
|
||||
tp_span_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
|
||||
tp_span_node2 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
|
||||
|
||||
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
|
||||
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
|
||||
|
||||
|
||||
def test_daemon_rejects_unmappable_parallelism():
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import (
|
||||
_reject_unsupported_parallelism,
|
||||
)
|
||||
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
|
||||
)
|
||||
)
|
||||
# DP combines with --nnodes when the world size divides evenly.
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
)
|
||||
with pytest.raises(ValueError, match="pipeline"):
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
|
||||
)
|
||||
with pytest.raises(ValueError, match="exceeds"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
|
||||
key = WeightCacheKey(
|
||||
checkpoint="ckpt",
|
||||
model_arch="Arch",
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
dtype="bf16",
|
||||
quantization=None,
|
||||
quant_config_hash="h",
|
||||
revision=None,
|
||||
vllm_version="v",
|
||||
dp_size=16,
|
||||
dp_rank=3,
|
||||
)
|
||||
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
|
||||
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
|
||||
|
||||
Reference in New Issue
Block a user