From 24928a4d62d4d317ba070d219ce0ab6c3e58bae4 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 25 Sep 2026 14:56:39 +0800 Subject: [PATCH] [UX][Frontend] Introduce `vllm preload` cli for fast restart (#56680) Signed-off-by: Isotr0py --- docs/cli/README.md | 29 +- docs/features/preload.md | 203 ++++++++++++++ docs/mkdocs/gen_files/generate_argparse.py | 2 + .../model_loader/test_weight_cache.py | 3 +- vllm/config/load.py | 5 +- vllm/entrypoints/cli/main.py | 2 + vllm/entrypoints/cli/preload.py | 252 ++++++++++++++++++ .../model_loader/weight_cache/daemon.py | 193 +------------- 8 files changed, 496 insertions(+), 193 deletions(-) create mode 100644 docs/features/preload.md create mode 100644 vllm/entrypoints/cli/preload.py diff --git a/docs/cli/README.md b/docs/cli/README.md index 274dac2d8f..27425cd92c 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -9,7 +9,7 @@ vllm --help Available Commands: ```bash -vllm {chat,complete,serve,launch,bench,collect-env,run-batch} +vllm {chat,complete,serve,launch,bench,collect-env,run-batch,preload} ``` ## serve @@ -212,6 +212,33 @@ vllm run-batch \ See [vllm run-batch](./run-batch.md) for the full reference of all available arguments. +## preload + +Launch weight cache daemons (one per GPU) that hold the post-quantized, +TP-sharded weights in GPU memory and serve CUDA IPC handles to vLLM engines +over a Unix domain socket. Restarting engines then map the weights via +zero-copy IPC instead of reloading from disk, enabling fast engine restarts. + +```bash +# Launch one daemon per GPU +vllm preload --model meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4 + +# Engines then load from the daemons +vllm serve meta-llama/Llama-3.2-1B-Instruct --tensor-parallel-size 4 \ + --load-format ipc_cache +``` + +The daemon accepts the standard engine arguments (model, dtype, quantization, +tensor-parallel-size, ...) plus `--weight-cache-socket-dir` to override the +directory holding the per-GPU Unix sockets, and `--weight-cache-master-port` / +`--weight-cache-draft-master-port` to pin the daemon rendezvous ports for +multi-node and speculative-decoding setups. Tensor, expert and data +parallelism are supported; pipeline parallelism is rejected at launch. + +See [Preload](../features/preload.md) for how it works, cache modes, and +limitations, and [vllm preload](./preload.md) for the full reference of all +available arguments. + ## More Help For detailed options of any subcommand, use: diff --git a/docs/features/preload.md b/docs/features/preload.md new file mode 100644 index 0000000000..74839bf35b --- /dev/null +++ b/docs/features/preload.md @@ -0,0 +1,203 @@ +# Preload + +vLLM's preload feature keeps model artifacts resident in GPU memory across +engine restarts, so a restarting engine reuses them instead of rebuilding them +from scratch. The design is extensible to different kinds of artifacts; it +currently supports **model weights** via the weight cache daemon. + +With weight preloading, a daemon process per GPU holds its rank's +post-quantized, TP-sharded weights and serves CUDA IPC handles to vLLM engines +over a Unix domain socket, so a restarting engine maps the weights via +zero-copy IPC instead of reloading them from disk. + +Key benefits: + +- **Fast engine restarts**: weight loading reduces to mapping existing GPU + memory, cutting restart time from minutes (disk load + quantization + processing) to seconds. +- **Zero-copy sharing**: in the default `zero_copy` mode the engine directly + shares the daemon's GPU memory, so a restart adds no extra GPU memory + footprint. +- **Safe fallback**: if the daemon is unavailable or its cached weights don't + match the engine's configuration, the engine falls back to loading from + disk. + +!!! note + Weight preloading is only supported on CUDA and ROCm platforms. Tensor, + expert and data parallelism are supported, including across nodes; + pipeline parallelism is rejected when launching the daemon. + +## Quick start + +Launch one weight cache daemon per GPU with a single command: + +```bash +vllm preload --model meta-llama/Llama-3.1-8B-Instruct --tensor-parallel-size 4 +``` + +The daemon process loads and post-processes the weights once, then waits for +engines. Each rank binds its Unix socket only after its shard is fully cached, +so engines connecting before that simply fall back to disk loading. + +Then start (or restart) engines with the `ipc_cache` load format: + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct \ + --tensor-parallel-size 4 \ + --load-format ipc_cache +``` + +The daemon itself must load from disk; passing `--load-format ipc_cache` to +`vllm preload` is an error. + +## How it works + +1. `vllm preload` spawns one daemon process per GPU. Each daemon loads its + shard from disk using the configured loader, runs quantization + post-processing, and exports every parameter/buffer as a CUDA IPC handle. +2. An engine started with `--load-format ipc_cache` builds its model on the + meta device (no storage) and requests the tensors from the daemon on its + GPU. +3. Before serving anything, the engine and daemon compare a fingerprint of the + cached weights: checkpoint content (hashed from safetensors metadata, so + identical weights in different directories still match), model + architecture, TP/DP size/rank, dtype, quantization method and config, model + revision, and vLLM version. On any mismatch the engine falls back to disk + loading (unless `fallback` is disabled). +4. The engine also verifies the daemon's GPU UUID matches its own device, so a + stale socket cannot serve weights for the wrong GPU. + +Tied weights (e.g. `lm_head.weight` sharing storage with +`embed_tokens.weight`) are exported once and re-established as aliases in the +engine, preserving parameter identity. + +## Multi-node and data parallelism + +CUDA IPC handles are node-local, so each node serves only its local GPUs' +shards. For multi-node tensor parallelism, run one `vllm preload` launcher per +node with a shared rendezvous: reuse the `--nnodes` / `--node-rank` / +`--master-addr` flags you pass the engine, plus a `--weight-cache-master-port` +distinct from the engine's `--master-port`: + +```bash +# node 0 (8 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 16 \ + --nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \ + --weight-cache-master-port 29600 +# node 1 (8 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 16 \ + --nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \ + --weight-cache-master-port 29600 +``` + +For data parallelism (e.g. a TP1 x DP16 x EP decode fleet), run one launcher +per node with the engine's DP placement flags. Local GPU `i` serves DP rank +`start_rank + i // tp_size` and TP rank `i % tp_size`, and all +`dp_size * tp_size` daemons form one world group on `--data-parallel-address` +/ `--weight-cache-master-port` so the expert shards are laid out exactly as in +the engine: + +```bash +# node r (4 local GPUs) +vllm preload --model /path/to/model --tensor-parallel-size 1 \ + --enable-expert-parallel \ + --data-parallel-size 16 --data-parallel-size-local 4 \ + --data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \ + --weight-cache-master-port 29600 +``` + +Data parallelism also combines with multi-node tensor parallelism: pass both +flag sets. Each node then serves a contiguous block of the +`dp_size * tp_size` global ranks. + +## Speculative decoding + +With MTP, EAGLE or EAGLE3 speculative decoding, `vllm preload` additionally +starts a draft daemon group that caches the draft model. It uses its own cache +key, Unix sockets (`*_draft.sock`) and rendezvous port +(`--weight-cache-draft-master-port`, default `--weight-cache-master-port + 1`), +so each daemon process serves exactly one model role. Other draft types are +not cached and keep loading from disk in the engine. + +## Cache modes + +The loader supports two modes, selected via `--model-loader-extra-config`: + +- `zero_copy` (default): the engine maps the daemon's CUDA allocations + directly. The daemon must stay alive for the engine's lifetime, and each + GPU carries only one copy of the weights. +- `copy`: the engine clones every tensor into its own GPU memory and then + asks the daemon to release its cache. Use this when the daemon should free + GPU memory after handing off, at the cost of a full copy per restart. + +!!! warning + In `zero_copy` mode the weights live in the daemon's CUDA IPC allocations, + so [sleep mode](sleep_mode.md) (CuMemAllocator weight offloading) must not + be used with this loader. + +## Loader configuration + +The `ipc_cache` loader accepts extra keys via `--model-loader-extra-config`: + +```bash +vllm serve meta-llama/Llama-3.1-8B-Instruct \ + --load-format ipc_cache \ + --model-loader-extra-config '{"mode": "copy", "fallback": false}' +``` + +| Key | Default | Description | +| --- | ------- | ----------- | +| `socket_path` | per-GPU path derived from the GPU UUID | Explicit daemon socket path. | +| `socket_dir` | per-user private dir under the temp dir | Directory containing the daemon sockets. | +| `mode` | `zero_copy` | `zero_copy` or `copy` (see above). | +| `fallback` | `true` | Fall back to disk loading when the daemon is unavailable or the fingerprints mismatch. | +| `connect_timeout_s` | `5.0` | Socket connect timeout in seconds. | +| `state_timeout_s` | `300.0` | Timeout for the weight-transfer request in seconds. | + +The daemon's `--weight-cache-socket-dir` and the loader's `socket_dir` / +`socket_path` must agree when the default per-user directory is not used. +Socket paths are derived from the GPU UUID, so they are stable regardless of +`CUDA_VISIBLE_DEVICES` index remapping. + +## Limitations + +- **Platform**: CUDA and ROCm only; other platforms raise + `UnsupportedPlatformForIPCError` even when `fallback` is enabled, since it + is a permanent misconfiguration rather than a transient daemon outage. +- **Parallelism**: tensor, expert and data parallelism are supported; + launching the daemon with pipeline parallelism is rejected. +- **Quantization**: every quantization method in the model must declare + support for pre-processed weights (the daemon transfers weights *after* + quantization post-processing). Unsupported methods raise + `UnsupportedQuantForIPCError`. +- **Consistency**: the daemon and engine must run the same vLLM version and + agree on model, dtype, quantization, and TP/DP layout, otherwise the + fingerprint mismatch triggers the disk fallback. +- **Localhost only**: the daemon serves over a Unix domain socket; both the + daemon and the engines must run on the same node as the same user. + +## Security + +The socket protocol uses pickle and is intended only for trusted local +processes owned by the same user: + +- Daemon sockets live in a per-user private directory (mode `0700`) and the + socket files are restricted to the owner (`0600`). +- Both sides reject symlinked or non-owned socket paths; the auto-derived + directory is additionally rejected if it is group/world accessible. +- On Linux the daemon verifies the connecting peer's UID via `SO_PEERCRED`. + +See the [security documentation](../usage/security.md) for vLLM's general +threat model. + +## Daemon lifecycle + +- Each GPU's socket path is guarded by an exclusive lock file, so a second + daemon for the same GPU fails fast instead of clobbering the live socket. +- `vllm preload` reports readiness once every rank is serving; if any rank + dies during startup, the remaining ranks are terminated and the command + exits with that rank's exit code. +- `SIGINT`/`SIGTERM` to the `vllm preload` process terminates all daemon + ranks, which unlinks their sockets and releases the GPU memory. + +See [vllm preload](../cli/preload.md) for the full CLI reference. diff --git a/docs/mkdocs/gen_files/generate_argparse.py b/docs/mkdocs/gen_files/generate_argparse.py index ecdce7761f..a45de12f2e 100644 --- a/docs/mkdocs/gen_files/generate_argparse.py +++ b/docs/mkdocs/gen_files/generate_argparse.py @@ -177,6 +177,7 @@ BenchmarkMMProcessorSubcommand = auto_mock( LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase") launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION") RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand") +PreloadSubcommand = auto_mock("vllm.entrypoints.cli.preload", "PreloadSubcommand") sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS") openai_cli_args = auto_mock("vllm.entrypoints.launchers", "cli_args") run_batch = auto_mock("vllm.entrypoints.launchers", "run_batch") @@ -330,6 +331,7 @@ pages = { "cli/chat.md": (create_parser(ChatCommand.add_cli_args), False), "cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False), "cli/run-batch.md": (create_parser(run_batch.make_arg_parser), True), + "cli/preload.md": (create_parser(PreloadSubcommand.add_cli_args), True), "cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True), "cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True), # URL kept as `mm_processor` for back-compat; command name is `mm-processor` diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 2ea77e79c8..2280fff8b7 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -36,7 +36,8 @@ class WeightCacheDaemon: self._cmd = [ sys.executable, "-m", - "vllm.model_executor.model_loader.weight_cache.daemon", + "vllm.entrypoints.cli.main", + "preload", "--model", model, "--tensor-parallel-size", diff --git a/vllm/config/load.py b/vllm/config/load.py index ee13e2fccd..d82997be17 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -39,9 +39,8 @@ class LoadConfig: InstantTensor, which enables distributed loading with pipelined prefetching and fast direct I/O. - "ipc_cache" will map post-quantized weights from a local weight cache - daemon via CUDA IPC for fast engine restarts. See - `vllm/model_executor/model_loader/weight_cache/daemon.py` for how to - launch the daemon. + daemon via CUDA IPC for fast engine restarts. Launch the daemon with + `vllm preload` first. - "npcache" will load the weights in pytorch format and store a numpy cache to speed up the loading. - "dummy" will initialize the weights with random values, which is mainly diff --git a/vllm/entrypoints/cli/main.py b/vllm/entrypoints/cli/main.py index ab9afcb008..5a8a82836b 100644 --- a/vllm/entrypoints/cli/main.py +++ b/vllm/entrypoints/cli/main.py @@ -44,6 +44,7 @@ def main(): import vllm.entrypoints.cli.collect_env import vllm.entrypoints.cli.launch import vllm.entrypoints.cli.openai + import vllm.entrypoints.cli.preload import vllm.entrypoints.cli.run_batch import vllm.entrypoints.cli.serve import vllm.entrypoints.cli.snapshot @@ -59,6 +60,7 @@ def main(): vllm.entrypoints.cli.launch, vllm.entrypoints.cli.benchmark.main, vllm.entrypoints.cli.collect_env, + vllm.entrypoints.cli.preload, vllm.entrypoints.cli.run_batch, vllm.entrypoints.cli.snapshot, ] diff --git a/vllm/entrypoints/cli/preload.py b/vllm/entrypoints/cli/preload.py new file mode 100644 index 0000000000..d312b6c36a --- /dev/null +++ b/vllm/entrypoints/cli/preload.py @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The `preload` subcommand for the vLLM CLI. + +Launches one weight cache daemon per GPU; each daemon holds the +post-quantized, TP-sharded weights of its rank in GPU memory and serves +CUDA IPC handles to vLLM engines over a Unix domain socket, so restarting +engines can map the weights via zero-copy IPC instead of reloading from disk: + + vllm preload --model /path/to/model --tensor-parallel-size 4 + +Engines then load from the daemons with: + + vllm serve /path/to/model --tensor-parallel-size 4 --load-format ipc_cache + +Tensor, expert and data parallelism are supported, including across nodes; +see vllm/model_executor/model_loader/weight_cache/daemon.py for the rank +layout and rendezvous details. +""" + +import argparse +import multiprocessing +import queue +import signal +import sys +import typing +from itertools import product + +from vllm.engine.arg_utils import EngineArgs +from vllm.entrypoints.cli.types import CLISubcommand +from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG +from vllm.logger import init_logger +from vllm.model_executor.model_loader.weight_cache.protocol import ( + check_ipc_platform_support, +) +from vllm.utils.network_utils import get_distributed_init_method, get_open_port + +if typing.TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.utils.argparse_utils import FlexibleArgumentParser +else: + FlexibleArgumentParser = argparse.ArgumentParser + +logger = init_logger(__name__) + + +class PreloadSubcommand(CLISubcommand): + """The `preload` subcommand for the vLLM CLI.""" + + name = "preload" + + @staticmethod + def add_cli_args(parser: FlexibleArgumentParser) -> None: + EngineArgs.add_cli_args(parser) + parser.add_argument( + "--weight-cache-socket-dir", + type=str, + default=None, + help="Directory for the daemon Unix sockets (default: tempdir).", + ) + parser.add_argument( + "--weight-cache-master-port", + type=int, + default=None, + help="Rendezvous port for the daemon's own TP group. Must differ " + "from the engine's --master-port (the daemon holds its group open " + "while serving) and match across nodes. Required when --nnodes > " + "1; defaults to a free port for single-node.", + ) + parser.add_argument( + "--weight-cache-draft-master-port", + type=int, + default=None, + help="Rendezvous port for the MTP draft daemon group. Defaults to " + "--weight-cache-master-port + 1 for multi-node, or a free port " + "for single-node.", + ) + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + # Imported here so `vllm --help` and friends stay free of torch. + from vllm.model_executor.model_loader.weight_cache.daemon import ( + _reject_unsupported_parallelism, + _run_daemon, + get_draft_daemon_config, + plan_local_ranks, + ) + from vllm.model_executor.model_loader.weight_cache.utils import ( + format_daemon_role, + ) + + engine_args = EngineArgs.from_cli_args(args) + vllm_config = engine_args.create_engine_config() + if vllm_config.load_config.load_format == "ipc_cache": + raise ValueError( + "The weight cache daemon itself must load from disk; use the " + "default --load-format" + ) + # Config-only so it can fail before any model loading; the quant + # method check needs the created model and runs in + # WeightCacheDaemon.get_model. + check_ipc_platform_support() + parallel_config = vllm_config.parallel_config + _reject_unsupported_parallelism(parallel_config) + tp_size = parallel_config.tensor_parallel_size + dp_size = parallel_config.data_parallel_size + + placements = plan_local_ranks(parallel_config) + local_world_size = len(placements) + if dp_size == 1: + nnodes = parallel_config.nnodes + node_rank = parallel_config.node_rank + master_addr = parallel_config.master_addr + else: + master_addr = parallel_config.data_parallel_master_ip + if parallel_config.nnodes > 1: + nnodes = parallel_config.nnodes + node_rank = parallel_config.node_rank + else: + nnodes = dp_size // parallel_config.data_parallel_size_local + node_rank = parallel_config.data_parallel_rank // ( + parallel_config.data_parallel_size_local + ) + if nnodes > 1 and master_addr in ("127.0.0.1", "localhost"): + raise ValueError( + "Data parallelism across nodes requires a reachable " + "--data-parallel-address for the daemon rendezvous" + ) + + # The daemon forms its own world group and holds it open while + # serving, so it needs a rendezvous port distinct from the engine's. + # All nodes must agree on it; single-node can auto-pick a free port. + # The master address is the engine's --master-addr (TP across nodes) + # or --data-parallel-address. + if nnodes > 1 and args.weight_cache_master_port is None: + raise ValueError( + "--weight-cache-master-port is required when the daemons span nodes" + ) + master_port = args.weight_cache_master_port or get_open_port() + distributed_init_method = get_distributed_init_method(master_addr, master_port) + + # (is_draft, vllm_config, rendezvous) per daemon group. + groups: list[tuple[bool, VllmConfig, str]] = [ + (False, vllm_config, distributed_init_method) + ] + draft_vllm_config = get_draft_daemon_config(vllm_config) + if draft_vllm_config is not None: + draft_master_port = args.weight_cache_draft_master_port or ( + master_port + 1 if nnodes > 1 else get_open_port() + ) + if draft_master_port == master_port: + raise ValueError( + "--weight-cache-draft-master-port must differ from " + "--weight-cache-master-port" + ) + groups.append( + ( + True, + draft_vllm_config, + get_distributed_init_method(master_addr, draft_master_port), + ) + ) + + ctx = multiprocessing.get_context("spawn") + ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue() + # Local index == device index; global rank enumerates DP then TP. + expected_ready = { + (format_daemon_role(is_draft), dp_rank * tp_size + tp_rank) + for (is_draft, _, _), (_, dp_rank, tp_rank) in product(groups, placements) + } + procs = [ + ctx.Process( + target=_run_daemon, + args=( + tp_rank, + local_rank, + config, + init_method, + args.weight_cache_socket_dir, + ready_queue, + is_draft, + dp_rank, + ), + name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-" + f"{dp_rank * tp_size + tp_rank}", + ) + for (is_draft, config, init_method), ( + local_rank, + dp_rank, + tp_rank, + ) in product(groups, placements) + ] + for proc in procs: + proc.start() + + def _shutdown(signum, frame): + for proc in procs: + proc.terminate() + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + + ready: set[tuple[str, int]] = set() + while len(ready) < len(expected_ready): + try: + ready.add(ready_queue.get(timeout=1.0)) + except queue.Empty: + dead = [p for p in procs if p.exitcode is not None] + if dead: + logger.error( + "Weight cache daemon rank(s) exited during startup " + "(exitcodes=%s); shutting down.", + [p.exitcode for p in dead], + ) + for proc in procs: + proc.terminate() + for proc in procs: + proc.join() + sys.exit(max((p.exitcode or 0) for p in procs)) + socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir" + logger.info_once( + "===== Weight cache daemon READY: node %d/%d serving %d local " + "rank(s) x %d role(s) in %s =====", + node_rank, + nnodes, + local_world_size, + len(groups), + socket_dir_msg, + ) + + for proc in procs: + proc.join() + sys.exit(max(proc.exitcode or 0 for proc in procs)) + + def subparser_init( + self, subparsers: argparse._SubParsersAction + ) -> FlexibleArgumentParser: + preload_parser = subparsers.add_parser( + self.name, + help="Launch weight cache daemons (one per GPU) for fast engine restarts.", + description="Launch weight cache daemons (one per GPU) that " + "keep post-quantized weights in GPU memory and serve them to " + "engines over CUDA IPC.", + usage="vllm preload --model [options]", + ) + self.add_cli_args(preload_parser) + preload_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name) + return preload_parser + + +def cmd_init() -> list[CLISubcommand]: + return [PreloadSubcommand()] diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index 70fb31d579..561f693ae7 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -9,7 +9,7 @@ reloading from disk. Launch one daemon per TP rank with a single command: - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 4 Engines then load from the daemons with: @@ -27,12 +27,12 @@ node-local, so each node serves only its local GPUs' shards). Reuse the same ``--weight-cache-master-port`` distinct from the engine's ``--master-port``: # node 0 (8 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 16 \\ --nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \\ --weight-cache-master-port 29600 # node 1 (8 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 16 \\ --nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \\ --weight-cache-master-port 29600 @@ -50,7 +50,7 @@ node with the engine's DP placement flags. Local GPU ``i`` serves DP rank shards are laid out exactly as in the engine: # node r (4 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 1 --enable-expert-parallel \\ --data-parallel-size 16 --data-parallel-size-local 4 \\ --data-parallel-start-rank 4r --data-parallel-address 10.0.0.1 \\ @@ -62,7 +62,7 @@ a reachable ``--data-parallel-address``). Each node then serves a contiguous block of the ``dp_size * tp_size`` global ranks, e.g. TP8 x DP2 on 4 nodes: # node r (4 local GPUs) - python -m vllm.model_executor.model_loader.weight_cache.daemon \\ + vllm preload \\ --model /path/to/model --tensor-parallel-size 8 --enable-expert-parallel \\ --nnodes 4 --node-rank r --master-addr 10.0.0.1 \\ --data-parallel-size 2 --data-parallel-address 10.0.0.1 \\ @@ -80,12 +80,8 @@ import contextlib import fcntl import multiprocessing import os -import queue -import signal import socket -import sys from collections.abc import Callable -from itertools import product import torch @@ -99,7 +95,6 @@ from vllm.distributed import ( ensure_model_parallel_initialized, init_distributed_environment, ) -from vllm.engine.arg_utils import EngineArgs from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model_loader from vllm.model_executor.model_loader.utils import process_weights_after_loading @@ -107,7 +102,6 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( TensorEntry, WeightCacheKey, WeightCacheUnavailableError, - check_ipc_platform_support, check_ipc_quant_support, ensure_private_socket_dir, get_current_device_uuid, @@ -122,8 +116,6 @@ from vllm.model_executor.model_loader.weight_cache.utils import ( is_draft_model_cacheable, ) from vllm.platforms import current_platform -from vllm.utils.argparse_utils import FlexibleArgumentParser -from vllm.utils.network_utils import get_distributed_init_method, get_open_port from vllm.utils.torch_utils import set_default_torch_dtype from vllm.v1.worker.workspace import init_workspace_manager @@ -504,178 +496,3 @@ def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None: "--data-parallel-start-rank + --data-parallel-size-local " f"({end_rank}) exceeds --data-parallel-size ({dp_size})" ) - - -def main() -> None: - parser = FlexibleArgumentParser( - description="Launch weight cache daemons (one per TP rank)." - ) - EngineArgs.add_cli_args(parser) - parser.add_argument( - "--weight-cache-socket-dir", - type=str, - default=None, - help="Directory for the daemon Unix sockets (default: tempdir).", - ) - parser.add_argument( - "--weight-cache-master-port", - type=int, - default=None, - help="Rendezvous port for the daemon's own TP group. Must differ from " - "the engine's --master-port (the daemon holds its group open while " - "serving) and match across nodes. Required when --nnodes > 1; defaults " - "to a free port for single-node.", - ) - parser.add_argument( - "--weight-cache-draft-master-port", - type=int, - default=None, - help="Rendezvous port for the MTP draft daemon group. Defaults to " - "--weight-cache-master-port + 1 for multi-node, or a free port for " - "single-node.", - ) - args = parser.parse_args() - engine_args = EngineArgs.from_cli_args(args) - vllm_config = engine_args.create_engine_config() - if vllm_config.load_config.load_format == "ipc_cache": - raise ValueError( - "The weight cache daemon itself must load from disk; use the " - "default --load-format" - ) - # Config-only so it can fail before any model loading; the quant method - # check needs the created model and runs in get_daemon_model. - check_ipc_platform_support() - parallel_config = vllm_config.parallel_config - _reject_unsupported_parallelism(parallel_config) - tp_size = parallel_config.tensor_parallel_size - dp_size = parallel_config.data_parallel_size - - placements = plan_local_ranks(parallel_config) - local_world_size = len(placements) - if dp_size == 1: - nnodes = parallel_config.nnodes - node_rank = parallel_config.node_rank - master_addr = parallel_config.master_addr - else: - master_addr = parallel_config.data_parallel_master_ip - if parallel_config.nnodes > 1: - nnodes = parallel_config.nnodes - node_rank = parallel_config.node_rank - else: - nnodes = dp_size // parallel_config.data_parallel_size_local - node_rank = parallel_config.data_parallel_rank // ( - parallel_config.data_parallel_size_local - ) - if nnodes > 1 and master_addr in ("127.0.0.1", "localhost"): - raise ValueError( - "Data parallelism across nodes requires a reachable " - "--data-parallel-address for the daemon rendezvous" - ) - - # The daemon forms its own world group and holds it open while serving, so - # it needs a rendezvous port distinct from the engine's. All nodes must - # agree on it; single-node can auto-pick a free port. The master address is - # the engine's --master-addr (TP across nodes) or --data-parallel-address. - if nnodes > 1 and args.weight_cache_master_port is None: - raise ValueError( - "--weight-cache-master-port is required when the daemons span nodes" - ) - master_port = args.weight_cache_master_port or get_open_port() - distributed_init_method = get_distributed_init_method(master_addr, master_port) - - # (is_draft, vllm_config, rendezvous) per daemon group. - groups: list[tuple[bool, VllmConfig, str]] = [ - (False, vllm_config, distributed_init_method) - ] - draft_vllm_config = get_draft_daemon_config(vllm_config) - if draft_vllm_config is not None: - draft_master_port = args.weight_cache_draft_master_port or ( - master_port + 1 if nnodes > 1 else get_open_port() - ) - if draft_master_port == master_port: - raise ValueError( - "--weight-cache-draft-master-port must differ from " - "--weight-cache-master-port" - ) - groups.append( - ( - True, - draft_vllm_config, - get_distributed_init_method(master_addr, draft_master_port), - ) - ) - - ctx = multiprocessing.get_context("spawn") - ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue() - # Local index == device index; global rank enumerates DP then TP. - expected_ready = { - (format_daemon_role(is_draft), dp_rank * tp_size + tp_rank) - for (is_draft, _, _), (_, dp_rank, tp_rank) in product(groups, placements) - } - procs = [ - ctx.Process( - target=_run_daemon, - args=( - tp_rank, - local_rank, - config, - init_method, - args.weight_cache_socket_dir, - ready_queue, - is_draft, - dp_rank, - ), - name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-" - f"{dp_rank * tp_size + tp_rank}", - ) - for (is_draft, config, init_method), ( - local_rank, - dp_rank, - tp_rank, - ) in product(groups, placements) - ] - for proc in procs: - proc.start() - - def _shutdown(signum, frame): - for proc in procs: - proc.terminate() - - signal.signal(signal.SIGINT, _shutdown) - signal.signal(signal.SIGTERM, _shutdown) - - ready: set[tuple[str, int]] = set() - while len(ready) < len(expected_ready): - try: - ready.add(ready_queue.get(timeout=1.0)) - except queue.Empty: - dead = [p for p in procs if p.exitcode is not None] - if dead: - logger.error( - "Weight cache daemon rank(s) exited during startup " - "(exitcodes=%s); shutting down.", - [p.exitcode for p in dead], - ) - for proc in procs: - proc.terminate() - for proc in procs: - proc.join() - sys.exit(max((p.exitcode or 0) for p in procs)) - socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir" - logger.info_once( - "===== Weight cache daemon READY: node %d/%d serving %d local rank(s) " - "x %d role(s) in %s =====", - node_rank, - nnodes, - local_world_size, - len(groups), - socket_dir_msg, - ) - - for proc in procs: - proc.join() - sys.exit(max(proc.exitcode or 0 for proc in procs)) - - -if __name__ == "__main__": - main()