[Fast Start] Fast loader support nnode>1 (#55468)

Signed-off-by: siyu <mg21330037@smail.nju.edu.cn>
Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
siyu
2026-09-14 03:05:54 +00:00
committed by GitHub
co-authored by Isotr0py
parent eb42686a30
commit 52dd0d7562
@@ -17,8 +17,30 @@ Engines then load from the daemons with:
vllm serve /path/to/model --tensor-parallel-size 4 \\
--load-format ipc_cache
Only tensor and expert parallelism are supported; pipeline and data
parallelism are rejected at launch.
Only tensor and expert parallelism are supported; pipeline and data parallelism
are rejected at launch.
For multi-node tensor parallelism, run one launcher per node with a shared
rendezvous so the global TP group forms across nodes (CUDA IPC handles are
node-local, so each node serves only its local GPUs' shards). Reuse the same
``--nnodes``/``--node-rank``/``--master-addr`` flags you pass the engine, plus a
``--weight-cache-master-port`` distinct from the engine's ``--master-port``:
# node 0 (8 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
--model /path/to/model --tensor-parallel-size 16 \\
--nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \\
--weight-cache-master-port 29600
# node 1 (8 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
--model /path/to/model --tensor-parallel-size 16 \\
--nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \\
--weight-cache-master-port 29600
The global TP rank of local GPU ``i`` on node ``r`` is
``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank
assignment, so each engine worker maps its shard from the daemon on its own
node.
"""
import contextlib
@@ -132,11 +154,13 @@ class WeightCacheDaemon:
self,
vllm_config: VllmConfig,
tp_rank: int,
local_rank: int,
distributed_init_method: str,
socket_dir: str | None = None,
):
self.vllm_config = vllm_config
self.tp_rank = tp_rank
self.local_rank = local_rank
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
self.model: torch.nn.Module | None = None
@@ -152,12 +176,12 @@ class WeightCacheDaemon:
def load_model(self) -> None:
tp_size = self.cache_config.tp_size
torch.accelerator.set_device_index(self.tp_rank)
torch.accelerator.set_device_index(self.local_rank)
init_distributed_environment(
world_size=tp_size,
rank=self.tp_rank,
distributed_init_method=self.distributed_init_method,
local_rank=self.tp_rank,
local_rank=self.local_rank,
backend=current_platform.dist_backend,
)
with set_current_vllm_config(self.vllm_config):
@@ -307,13 +331,14 @@ class WeightCacheDaemon:
def _run_daemon(
tp_rank: int,
local_rank: int,
vllm_config: VllmConfig,
distributed_init_method: str,
socket_dir: str | None,
ready_queue: "multiprocessing.Queue[int]",
) -> None:
daemon = WeightCacheDaemon(
vllm_config, tp_rank, distributed_init_method, socket_dir
vllm_config, tp_rank, local_rank, distributed_init_method, socket_dir
)
daemon.load_model()
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
@@ -348,6 +373,15 @@ def main() -> None:
default=None,
help="Directory for the daemon Unix sockets (default: tempdir).",
)
parser.add_argument(
"--weight-cache-master-port",
type=int,
default=None,
help="Rendezvous port for the daemon's own TP group. Must differ from "
"the engine's --master-port (the daemon holds its group open while "
"serving) and match across nodes. Required when --nnodes > 1; defaults "
"to a free port for single-node.",
)
args = parser.parse_args()
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
@@ -363,22 +397,41 @@ def main() -> None:
_reject_unsupported_parallelism(parallel_config)
tp_size = parallel_config.tensor_parallel_size
distributed_init_method = get_distributed_init_method("127.0.0.1", get_open_port())
nnodes = parallel_config.nnodes
node_rank = parallel_config.node_rank
local_world_size = tp_size // nnodes
# The daemon forms its own TP group and holds it open while serving, so it
# needs a rendezvous port distinct from the engine's --master-port. All
# nodes must agree on it; single-node can auto-pick a free port. The master
# address is reused from the engine's --master-addr (node_rank 0).
master_addr = parallel_config.master_addr
if nnodes > 1 and args.weight_cache_master_port is None:
raise ValueError("--weight-cache-master-port is required when --nnodes > 1")
master_port = args.weight_cache_master_port or get_open_port()
distributed_init_method = get_distributed_init_method(master_addr, master_port)
ctx = multiprocessing.get_context("spawn")
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
# Global TP rank of local GPU i on this node; local index == device index.
global_ranks = [
node_rank * local_world_size + local_rank
for local_rank in range(local_world_size)
]
procs = [
ctx.Process(
target=_run_daemon,
args=(
rank,
global_rank,
local_rank,
vllm_config,
distributed_init_method,
args.weight_cache_socket_dir,
ready_queue,
),
name=f"vllm-weight-cache-daemon-{rank}",
name=f"vllm-weight-cache-daemon-{global_rank}",
)
for rank in range(tp_size)
for local_rank, global_rank in enumerate(global_ranks)
]
for proc in procs:
proc.start()
@@ -391,7 +444,7 @@ def main() -> None:
signal.signal(signal.SIGTERM, _shutdown)
ready_ranks: set[int] = set()
while len(ready_ranks) < tp_size:
while len(ready_ranks) < local_world_size:
try:
ready_ranks.add(ready_queue.get(timeout=1.0))
except queue.Empty:
@@ -407,11 +460,16 @@ def main() -> None:
for proc in procs:
proc.join()
sys.exit(max((p.exitcode or 0) for p in procs))
logger.info(
"Weight cache daemon ready: all %d ranks serving in %s",
tp_size,
args.weight_cache_socket_dir or "the default socket dir",
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
logger.info_once(
"===== Weight cache daemon READY: node %d/%d serving %d local rank(s) "
"in %s =====",
node_rank,
nnodes,
local_world_size,
socket_dir_msg,
)
for proc in procs:
proc.join()
sys.exit(max(proc.exitcode or 0 for proc in procs))