From 567f745c1b14ffaf09bfa71b7c4d0177a7994c04 Mon Sep 17 00:00:00 2001 From: shaohuaxi Date: Mon, 14 Sep 2026 19:43:42 -0700 Subject: [PATCH] [Bugfix] Export weight-cache IPC tensors separately for each client (#56472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 子华 Signed-off-by: Isotr0py Co-authored-by: Codex Co-authored-by: Isotr0py --- .../model_loader/weight_cache/daemon.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index b19fb96272..f0af5a284a 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -93,11 +93,14 @@ def export_entries( ``named_parameters``/``named_buffers`` are iterated with ``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing storage with ``embed_tokens.weight``) are not silently dropped. Each unique - tensor is exported once; every additional name that refers to the same + tensor is exported once per call; every additional name that refers to the same tensor object is recorded in the returned alias map so the client can re-establish the shared identity instead of allocating uninitialized memory for it. + CUDA reduction arguments must be exported separately for each consumer so + that PyTorch registers a reference for each IPC mapping's lifetime. + Returns: A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to its ``TensorEntry`` and ``aliases`` maps each duplicate name to its @@ -167,8 +170,6 @@ class WeightCacheDaemon: self.distributed_init_method = distributed_init_method self.socket_dir = socket_dir self.model: torch.nn.Module | None = None - self.entries: dict[str, TensorEntry] = {} - self.aliases: dict[str, str] = {} # Fingerprint before loading: process_weights_after_loading may # mutate hf_config.quantization_config. self.cache_config = WeightCacheKey.from_model_config( @@ -190,17 +191,11 @@ class WeightCacheDaemon: with set_current_vllm_config(self.vllm_config): ensure_model_parallel_initialized(tp_size, 1) self.model = get_daemon_model(self.vllm_config) - self._export_entries() logger.info( - "Weight cache daemon rank %d cached %d tensors", + "Weight cache daemon rank %d loaded model", self.tp_rank, - len(self.entries), ) - def _export_entries(self) -> None: - assert self.model is not None - self.entries, self.aliases = export_entries(self.model) - def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None: """Serve requests until terminated. @@ -295,28 +290,28 @@ class WeightCacheDaemon: logger.warning("WeightCacheKey mismatch on fields: %s", mismatched) send_msg(conn, {"status": "mismatch", "fields": mismatched}) return - if not self.entries: + if self.model is None: send_msg(conn, {"status": "error", "message": "Weights were released"}) return + gpu_uuid = get_current_device_uuid() + entries, aliases = export_entries(self.model) send_msg( conn, { "status": "ok", - "entries": self.entries, - "aliases": self.aliases, - "gpu_uuid": get_current_device_uuid(), + "entries": entries, + "aliases": aliases, + "gpu_uuid": gpu_uuid, }, ) logger.info_once( "Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine", self.tp_rank, - len(self.entries), - len(self.aliases), + len(entries), + len(aliases), ) def _handle_release(self, conn: socket.socket) -> None: - self.entries.clear() - self.aliases.clear() self.model = None torch.accelerator.empty_cache() logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)