[Fast Start] Support data parallelism in the weight cache daemon (#57386)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-22 18:19:40 +00:00
committed by GitHub
co-authored by Isotr0py
parent ca831d1c55
commit c024cfdb02
4 changed files with 285 additions and 42 deletions
@@ -251,3 +251,116 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
assert restart_outputs == baseline_outputs
def _parallel(**kw):
from types import SimpleNamespace
base = dict(
tensor_parallel_size=1,
pipeline_parallel_size=1,
data_parallel_size=1,
data_parallel_size_local=1,
data_parallel_rank=0,
nnodes=1,
node_rank=0,
)
base.update(kw)
return SimpleNamespace(**base)
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
"""Each node serves a contiguous global-rank block (DP-major), so without
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
by --data-parallel-start-rank."""
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
data_parallel_size=4,
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
def test_daemon_rejects_unmappable_parallelism():
from vllm.model_executor.model_loader.weight_cache.daemon import (
_reject_unsupported_parallelism,
)
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
)
)
# DP combines with --nnodes when the world size divides evenly.
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
with pytest.raises(ValueError, match="pipeline"):
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
)
with pytest.raises(ValueError, match="exceeds"):
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
)
)
def test_weight_cache_key_distinguishes_dp_ranks():
from dataclasses import replace
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
key = WeightCacheKey(
checkpoint="ckpt",
model_arch="Arch",
tp_size=1,
tp_rank=0,
dtype="bf16",
quantization=None,
quant_config_hash="h",
revision=None,
vllm_version="v",
dp_size=16,
dp_rank=3,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]