[UX][Frontend] Introduce vllm preload cli for fast restart (#56680)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-25 06:56:39 +00:00
committed by GitHub
parent 5ff3bbfb08
commit 24928a4d62
8 changed files with 496 additions and 193 deletions
@@ -9,7 +9,7 @@ reloading from disk.
Launch one daemon per TP rank with a single command:
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
vllm preload \\
--model /path/to/model --tensor-parallel-size 4
Engines then load from the daemons with:
@@ -27,12 +27,12 @@ node-local, so each node serves only its local GPUs' shards). Reuse the same
``--weight-cache-master-port`` distinct from the engine's ``--master-port``:
# node 0 (8 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
vllm preload \\
--model /path/to/model --tensor-parallel-size 16 \\
--nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \\
--weight-cache-master-port 29600
# node 1 (8 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
vllm preload \\
--model /path/to/model --tensor-parallel-size 16 \\
--nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \\
--weight-cache-master-port 29600
@@ -50,7 +50,7 @@ node with the engine's DP placement flags. Local GPU ``i`` serves DP rank
shards are laid out exactly as in the engine:
# node r (4 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
vllm preload \\
--model /path/to/model --tensor-parallel-size 1 --enable-expert-parallel \\
--data-parallel-size 16 --data-parallel-size-local 4 \\
--data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \\
@@ -62,7 +62,7 @@ a reachable ``--data-parallel-address``). Each node then serves a contiguous
block of the ``dp_size * tp_size`` global ranks, e.g. TP8 x DP2 on 4 nodes:
# node r (4 local GPUs)
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
vllm preload \\
--model /path/to/model --tensor-parallel-size 8 --enable-expert-parallel \\
--nnodes 4 --node-rank r --master-addr 10.0.0.1 \\
--data-parallel-size 2 --data-parallel-address 10.0.0.1 \\
@@ -80,12 +80,8 @@ import contextlib
import fcntl
import multiprocessing
import os
import queue
import signal
import socket
import sys
from collections.abc import Callable
from itertools import product
import torch
@@ -99,7 +95,6 @@ from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.utils import process_weights_after_loading
@@ -107,7 +102,6 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
TensorEntry,
WeightCacheKey,
WeightCacheUnavailableError,
check_ipc_platform_support,
check_ipc_quant_support,
ensure_private_socket_dir,
get_current_device_uuid,
@@ -122,8 +116,6 @@ from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.platforms import current_platform
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.worker.workspace import init_workspace_manager
@@ -504,178 +496,3 @@ def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
"--data-parallel-start-rank + --data-parallel-size-local "
f"({end_rank}) exceeds --data-parallel-size ({dp_size})"
)
def main() -> None:
parser = FlexibleArgumentParser(
description="Launch weight cache daemons (one per TP rank)."
)
EngineArgs.add_cli_args(parser)
parser.add_argument(
"--weight-cache-socket-dir",
type=str,
default=None,
help="Directory for the daemon Unix sockets (default: tempdir).",
)
parser.add_argument(
"--weight-cache-master-port",
type=int,
default=None,
help="Rendezvous port for the daemon's own TP group. Must differ from "
"the engine's --master-port (the daemon holds its group open while "
"serving) and match across nodes. Required when --nnodes > 1; defaults "
"to a free port for single-node.",
)
parser.add_argument(
"--weight-cache-draft-master-port",
type=int,
default=None,
help="Rendezvous port for the MTP draft daemon group. Defaults to "
"--weight-cache-master-port + 1 for multi-node, or a free port for "
"single-node.",
)
args = parser.parse_args()
engine_args = EngineArgs.from_cli_args(args)
vllm_config = engine_args.create_engine_config()
if vllm_config.load_config.load_format == "ipc_cache":
raise ValueError(
"The weight cache daemon itself must load from disk; use the "
"default --load-format"
)
# Config-only so it can fail before any model loading; the quant method
# check needs the created model and runs in get_daemon_model.
check_ipc_platform_support()
parallel_config = vllm_config.parallel_config
_reject_unsupported_parallelism(parallel_config)
tp_size = parallel_config.tensor_parallel_size
dp_size = parallel_config.data_parallel_size
placements = plan_local_ranks(parallel_config)
local_world_size = len(placements)
if dp_size == 1:
nnodes = parallel_config.nnodes
node_rank = parallel_config.node_rank
master_addr = parallel_config.master_addr
else:
master_addr = parallel_config.data_parallel_master_ip
if parallel_config.nnodes > 1:
nnodes = parallel_config.nnodes
node_rank = parallel_config.node_rank
else:
nnodes = dp_size // parallel_config.data_parallel_size_local
node_rank = parallel_config.data_parallel_rank // (
parallel_config.data_parallel_size_local
)
if nnodes > 1 and master_addr in ("127.0.0.1", "localhost"):
raise ValueError(
"Data parallelism across nodes requires a reachable "
"--data-parallel-address for the daemon rendezvous"
)
# The daemon forms its own world group and holds it open while serving, so
# it needs a rendezvous port distinct from the engine's. All nodes must
# agree on it; single-node can auto-pick a free port. The master address is
# the engine's --master-addr (TP across nodes) or --data-parallel-address.
if nnodes > 1 and args.weight_cache_master_port is None:
raise ValueError(
"--weight-cache-master-port is required when the daemons span nodes"
)
master_port = args.weight_cache_master_port or get_open_port()
distributed_init_method = get_distributed_init_method(master_addr, master_port)
# (is_draft, vllm_config, rendezvous) per daemon group.
groups: list[tuple[bool, VllmConfig, str]] = [
(False, vllm_config, distributed_init_method)
]
draft_vllm_config = get_draft_daemon_config(vllm_config)
if draft_vllm_config is not None:
draft_master_port = args.weight_cache_draft_master_port or (
master_port + 1 if nnodes > 1 else get_open_port()
)
if draft_master_port == master_port:
raise ValueError(
"--weight-cache-draft-master-port must differ from "
"--weight-cache-master-port"
)
groups.append(
(
True,
draft_vllm_config,
get_distributed_init_method(master_addr, draft_master_port),
)
)
ctx = multiprocessing.get_context("spawn")
ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue()
# Local index == device index; global rank enumerates DP then TP.
expected_ready = {
(format_daemon_role(is_draft), dp_rank * tp_size + tp_rank)
for (is_draft, _, _), (_, dp_rank, tp_rank) in product(groups, placements)
}
procs = [
ctx.Process(
target=_run_daemon,
args=(
tp_rank,
local_rank,
config,
init_method,
args.weight_cache_socket_dir,
ready_queue,
is_draft,
dp_rank,
),
name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-"
f"{dp_rank * tp_size + tp_rank}",
)
for (is_draft, config, init_method), (
local_rank,
dp_rank,
tp_rank,
) in product(groups, placements)
]
for proc in procs:
proc.start()
def _shutdown(signum, frame):
for proc in procs:
proc.terminate()
signal.signal(signal.SIGINT, _shutdown)
signal.signal(signal.SIGTERM, _shutdown)
ready: set[tuple[str, int]] = set()
while len(ready) < len(expected_ready):
try:
ready.add(ready_queue.get(timeout=1.0))
except queue.Empty:
dead = [p for p in procs if p.exitcode is not None]
if dead:
logger.error(
"Weight cache daemon rank(s) exited during startup "
"(exitcodes=%s); shutting down.",
[p.exitcode for p in dead],
)
for proc in procs:
proc.terminate()
for proc in procs:
proc.join()
sys.exit(max((p.exitcode or 0) for p in procs))
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
logger.info_once(
"===== Weight cache daemon READY: node %d/%d serving %d local rank(s) "
"x %d role(s) in %s =====",
node_rank,
nnodes,
local_world_size,
len(groups),
socket_dir_msg,
)
for proc in procs:
proc.join()
sys.exit(max(proc.exitcode or 0 for proc in procs))
if __name__ == "__main__":
main()