[Bugfix] Export weight-cache IPC tensors separately for each client (#56472)

Signed-off-by: 子华 <huaxi.shx@alibaba-inc.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
shaohuaxi
2026-09-15 02:43:42 +00:00
committed by GitHub
co-authored by Codex Isotr0py
parent 17bc3e106a
commit 567f745c1b
@@ -93,11 +93,14 @@ def export_entries(
``named_parameters``/``named_buffers`` are iterated with
``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing
storage with ``embed_tokens.weight``) are not silently dropped. Each unique
tensor is exported once; every additional name that refers to the same
tensor is exported once per call; every additional name that refers to the same
tensor object is recorded in the returned alias map so the client can
re-establish the shared identity instead of allocating uninitialized
memory for it.
CUDA reduction arguments must be exported separately for each consumer so
that PyTorch registers a reference for each IPC mapping's lifetime.
Returns:
A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to
its ``TensorEntry`` and ``aliases`` maps each duplicate name to its
@@ -167,8 +170,6 @@ class WeightCacheDaemon:
self.distributed_init_method = distributed_init_method
self.socket_dir = socket_dir
self.model: torch.nn.Module | None = None
self.entries: dict[str, TensorEntry] = {}
self.aliases: dict[str, str] = {}
# Fingerprint before loading: process_weights_after_loading may
# mutate hf_config.quantization_config.
self.cache_config = WeightCacheKey.from_model_config(
@@ -190,17 +191,11 @@ class WeightCacheDaemon:
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(tp_size, 1)
self.model = get_daemon_model(self.vllm_config)
self._export_entries()
logger.info(
"Weight cache daemon rank %d cached %d tensors",
"Weight cache daemon rank %d loaded model",
self.tp_rank,
len(self.entries),
)
def _export_entries(self) -> None:
assert self.model is not None
self.entries, self.aliases = export_entries(self.model)
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
"""Serve requests until terminated.
@@ -295,28 +290,28 @@ class WeightCacheDaemon:
logger.warning("WeightCacheKey mismatch on fields: %s", mismatched)
send_msg(conn, {"status": "mismatch", "fields": mismatched})
return
if not self.entries:
if self.model is None:
send_msg(conn, {"status": "error", "message": "Weights were released"})
return
gpu_uuid = get_current_device_uuid()
entries, aliases = export_entries(self.model)
send_msg(
conn,
{
"status": "ok",
"entries": self.entries,
"aliases": self.aliases,
"gpu_uuid": get_current_device_uuid(),
"entries": entries,
"aliases": aliases,
"gpu_uuid": gpu_uuid,
},
)
logger.info_once(
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
self.tp_rank,
len(self.entries),
len(self.aliases),
len(entries),
len(aliases),
)
def _handle_release(self, conn: socket.socket) -> None:
self.entries.clear()
self.aliases.clear()
self.model = None
torch.accelerator.empty_cache()
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)